fsanyoto commited on
Commit
36a3973
·
verified ·
1 Parent(s): 463918b

Deploy AIOS web (React glide grid + FastAPI slice)

Browse files
RELEASES.json CHANGED
@@ -1,6 +1,12 @@
1
  {
2
- "current": "50dae147b (50dae147b20ab)",
3
  "releases": [
 
 
 
 
 
 
4
  {
5
  "version": "v86",
6
  "sha": "108bd7510dcd2",
 
1
  {
2
+ "current": "9507685b7700 (9507685b7700c)",
3
  "releases": [
4
+ {
5
+ "version": "v87",
6
+ "sha": "288b6b8793041",
7
+ "date": "2026-11-12",
8
+ "subject": "release(sdlc85): release lessons and log line"
9
+ },
10
  {
11
  "version": "v86",
12
  "sha": "108bd7510dcd2",
VERSION CHANGED
@@ -1 +1 @@
1
- 50dae147b (50dae147b20ab)
 
1
+ 9507685b7700 (9507685b7700c)
api/registry_source.py CHANGED
@@ -535,6 +535,7 @@ class CanonicalStore(BatchStore):
535
  return super().get_projection(name, drop=drop)
536
 
537
  def get(self, name, fresh=False):
 
538
  if name in self.docs and not fresh:
539
  return copy.deepcopy(self.docs[name])
540
  if name.endswith('_cohorts'):
@@ -557,10 +558,18 @@ class CanonicalStore(BatchStore):
557
  value['routes'], value['zones'] = [], []
558
  for index, route in saved['routes']:
559
  if isinstance(route, dict):
560
- route = dict(route, stops=self._selected_json(name, ['routes', index, 'stops'],
561
- array=True, id_key='customerPid'))
 
562
  value['routes'].append(route)
563
  value['zones'] = copy.deepcopy(saved['zones'])
 
 
 
 
 
 
 
564
  elif name == 'table_zones':
565
  value = {self.table_key: []}
566
  for batch in array_values(self._runtime, name, [self.table_key], drop=(
@@ -877,12 +886,14 @@ def _canonical_batches(session, cur, table_key, *, ids=None, needed=None, metric
877
  raw = [field for field in raw if field.get('key') in required]
878
  import routes_keychain
879
  import time
880
- paused = routes_keychain.odoo_paused(session.runtime)
881
  pool_key = ('product_pool', team) if table_key == 'product_data' else ('pool', team, agent)
882
  cached = session.runtime.pool_cache.get(pool_key)
 
 
 
883
  # One product assembly per drill: its store reads cost per call, not per row (sdlc85:23).
884
  size = None if table_key == 'product_data' else BATCH_ROWS
885
- if cached and (paused or time.time() - cached[0] < 900):
886
  import itertools
887
  def borrowed():
888
  source = (r for r in cached[1] if ids is None or r['pid'] in ids)
 
535
  return super().get_projection(name, drop=drop)
536
 
537
  def get(self, name, fresh=False):
538
+ from core import user_tables
539
  if name in self.docs and not fresh:
540
  return copy.deepcopy(self.docs[name])
541
  if name.endswith('_cohorts'):
 
558
  value['routes'], value['zones'] = [], []
559
  for index, route in saved['routes']:
560
  if isinstance(route, dict):
561
+ # A stop names a customer pid, so a product batch reads no stop membership.
562
+ route = dict(route, stops=[] if self.table_key == 'product_data' else self._selected_json(
563
+ name, ['routes', index, 'stops'], array=True, id_key='customerPid'))
564
  value['routes'].append(route)
565
  value['zones'] = copy.deepcopy(saved['zones'])
566
+ elif name == user_tables.ODOO_SCHEMA_SNAPSHOT_KEY and self.table_key == 'product_data':
567
+ # Product assembly reads the product.product schema only, never the whole snapshot.
568
+ meta_key = ('product_schema', name)
569
+ if meta_key not in self.metadata or fresh:
570
+ model = paths(self._runtime, name, [['models', 'product.product']])[0]
571
+ self.metadata[meta_key] = {'models': {'product.product': model}} if isinstance(model, dict) else {}
572
+ value = self.metadata[meta_key]
573
  elif name == 'table_zones':
574
  value = {self.table_key: []}
575
  for batch in array_values(self._runtime, name, [self.table_key], drop=(
 
886
  raw = [field for field in raw if field.get('key') in required]
887
  import routes_keychain
888
  import time
 
889
  pool_key = ('product_pool', team) if table_key == 'product_data' else ('pool', team, agent)
890
  cached = session.runtime.pool_cache.get(pool_key)
891
+ fresh = bool(cached) and time.time() - cached[0] < 900
892
+ # A fresh pool serves paused or not, so only a stale or absent one reads the connector state.
893
+ paused = not fresh and routes_keychain.odoo_paused(session.runtime)
894
  # One product assembly per drill: its store reads cost per call, not per row (sdlc85:23).
895
  size = None if table_key == 'product_data' else BATCH_ROWS
896
+ if cached and (paused or fresh):
897
  import itertools
898
  def borrowed():
899
  source = (r for r in cached[1] if ids is None or r['pid'] in ids)
api/routes_admin.py CHANGED
@@ -1717,9 +1717,13 @@ def _row_wall_blind_keys(key, session=None, nodes=(), reasons=None, verdicts=Non
1717
  return blind
1718
 
1719
 
1720
- def _clean_perms(v, governed_keys=None, session=None, stored=None, stale_out=None):
 
1721
  """Validate a whole `perms` block. FAIL-CLOSED, and LOUD rather than lenient.
1722
 
 
 
 
1723
  ⭐⭐ W46-T35 — `stored` AND `stale_out` ARE OPTIONAL AND THE DEFAULT IS THE OLD BEHAVIOUR.
1724
  `stored` is the block this account ALREADY holds, handed down by `put_perms` from the
1725
  registry read it has already made — not a second read at a second moment, which is what the
@@ -1807,7 +1811,9 @@ def _clean_perms(v, governed_keys=None, session=None, stored=None, stale_out=Non
1807
  # measured, not feared. `clean_filter_tree` takes bare strings and cannot see the flag,
1808
  # so the narrowing has to happen HERE, at the call.
1809
  # Reuse submitted verdicts in the picker vocabulary so each leaf is judged once.
1810
- fields_here = _module_fields(key, session=session, wall_verdicts=wall_verdicts)
 
 
1811
  valid_keys = {f["key"] for f in fields_here}
1812
  filter_keys = {f["key"] for f in fields_here if f.get("filterable")}
1813
  # ⭐⭐ W41-T09 / R8 — THE COLUMNS THE ADMIN-OWNER RULE TOOK OUT OF THE BUILDER, READ OFF
@@ -2240,6 +2246,25 @@ def _prune_to_module(tree, valid_keys):
2240
  return {"conj": tree.get("conj", "and"), "nodes": nodes} if nodes else None
2241
 
2242
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2243
  @router.get("/admin/users/{username}/perms")
2244
  def get_perms(username: str, session: Session = Depends(admin_gate)):
2245
  """This account's permission block plus the field schema the editor needs to render it."""
@@ -2250,26 +2275,19 @@ def get_perms(username: str, session: Session = Depends(admin_gate)):
2250
  # 404, never 403, because "that account exists in another company" is itself a leak.
2251
  raise err(404, "no_such_user", f"no account named {uname!r}")
2252
  import core.perm_scope as perm_scope
2253
- from copy import copy
2254
- from core import user_tables, shares, freeze_notice
2255
- from routes_customers import _AssemblyStore
2256
 
2257
  try:
2258
  shares.reconcile_database_grants(session.runtime, tenant=session.tenant)
2259
  except freeze_notice.DataInputFrozen:
2260
  raise
2261
  except Exception as exc:
2262
- raise err(503, "store_unavailable", "Access unavailable. Try again.") from exc
 
 
2263
 
2264
- # One read-only request snapshot: definitions, grants and provenance are shared by the
2265
- # picker and cascade. It never enters a write door or outlives this request. Global account
2266
- # writes and their required post-write readback below continue through the original store.
2267
- session = copy(session)
2268
- session._permission_roster = reg
2269
- try:
2270
- session.runtime = user_tables.lend_defs(_AssemblyStore(session.runtime, schema_only=True))
2271
- except Exception:
2272
- raise err(503, "schema_unavailable", "The permission field definitions are unavailable. Try again.")
2273
 
2274
  rec = reg[uname]
2275
  stored = rec.get("perms") or {}
@@ -2342,6 +2360,7 @@ def get_perms(username: str, session: Session = Depends(admin_gate)):
2342
  if _gone:
2343
  stale_hidden_by_module[k] = _gone
2344
  perms_out = {}
 
2345
  for k in governed:
2346
  e = stored.get(k)
2347
  # ⛔⛔ W36-T22 — THE DEFAULT IS `may_read`, NOT `may_access`, AND THE DIFFERENCE IS AN
@@ -2376,7 +2395,8 @@ def get_perms(username: str, session: Session = Depends(admin_gate)):
2376
  perm_scope.may_read(users._public(uname, rec), k, st=session.runtime))
2377
  if (str(k).startswith("ut_") and isinstance(e, dict)
2378
  and e.get("access") is True and not _default):
2379
- raise err(503, "store_unavailable", "Access unavailable. Try again.")
 
2380
  # ⭐⭐ W38-T19 — `metrics` RIDES EVERY ENTRY, BACKFILLED TO GRANTED ON A RECORD WRITTEN
2381
  # BEFORE THE KEY EXISTED. The wire is TOTAL on purpose: the client's own parse already
2382
  # reads an absent key as granted, so this line changes no rendering — what it changes is
@@ -2424,6 +2444,8 @@ def get_perms(username: str, session: Session = Depends(admin_gate)):
2424
  # `is_migrated` slip away from consuming `_fold_legacy_scope`'s one chance, which is
2425
  # the BU leak the guard above exists for.
2426
  "stale_hidden_by_module": stale_hidden_by_module,
 
 
2427
  # ⭐⭐ W46-T34 — `{module: {count, refused, status, reason}}`, the metric census
2428
  # beside the field list it summarises. `status` is `ok` | `none` | `unavailable`,
2429
  # and the third is the one that had no way to be said before: an empty metric list
@@ -2519,9 +2541,49 @@ def migrate_perms(body: dict = Body(default=None), session: Session = Depends(ad
2519
  if not report["reconciliation"]["ok"]]}
2520
 
2521
 
2522
- def _sync_database_shares(session, uname, before, after):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2523
  """R15/C5: explicit ut_* access grants editor authority; explicit off removes the share.
2524
 
 
 
 
2525
  Two doors answer two questions: the access rule decides WHICH rows, a `core.shares` `database`
2526
  grant decides IF the member sees and edits it, so the editor's toggle writes both. Re-saving
2527
  an enabled entry repairs old view-only grants. Other principals, owners and provenance stay
@@ -2561,10 +2623,25 @@ def _sync_database_shares(session, uname, before, after):
2561
  if now:
2562
  entries.append({**(held[0] if held else {}), "user": uname, "role": "edit"})
2563
  try:
2564
- shares.set_grants("database", key, entries, st=session.runtime,
2565
- granter=session.uname, expect=record["entries"])
2566
  except shares.GrantsChanged as exc:
2567
  raise err(409, "shares_changed", "Database sharing changed. Save access again.") from exc
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2568
 
2569
 
2570
  def _set_database_request_share(session, key, entries, held, snapshot):
@@ -2835,9 +2912,14 @@ def put_perms(username: str, body: dict = Body(default=None),
2835
  # control for it: a report that lives only in a client this ticket does not own would be a
2836
  # feature nobody ships [[two-lanes-one-contract-dead-feature]].
2837
  _stale = {}
 
 
 
2838
  cleaned = _clean_perms(body.get("perms"),
2839
  governed_keys={m["key"] for m in _mods}, session=session,
2840
- stored=reg[uname].get("perms"), stale_out=_stale) or {}
 
 
2841
  governed = {m["key"] for m in _mods}
2842
  opaque = {key: entry for key, entry in cleaned.items() if key not in governed}
2843
  known = _fold_legacy_scope(reg[uname],
@@ -2845,6 +2927,7 @@ def put_perms(username: str, body: dict = Body(default=None),
2845
  session=session)
2846
  _guard_bu_widening(reg[uname], known, bool(body.get("confirm_widen")))
2847
  cleaned = dict(known, **opaque)
 
2848
  users.set_access(uname, perms=cleaned)
2849
  fresh = _registry()
2850
  rec = fresh.get(uname) or {}
@@ -2853,9 +2936,17 @@ def put_perms(username: str, body: dict = Body(default=None),
2853
  # the wall is up when it is not — the one direction this must never fail in.
2854
  raise err(503, "store_unavailable",
2855
  "the permissions were not saved; the account is UNCHANGED")
2856
- _sync_database_shares(session, uname, reg[uname].get("perms") or {},
2857
- {key: entry for key, entry in (rec.get("perms") or {}).items()
2858
- if key in governed})
 
 
 
 
 
 
 
 
2859
  return {"username": uname, "perms": rec.get("perms") or {},
2860
  "perms_v": int(rec.get("perms_v") or 0),
2861
  # W46-T35 — `{module: [key, ...]}`, empty on an ordinary save. What the save cleared
 
1717
  return blind
1718
 
1719
 
1720
+ def _clean_perms(v, governed_keys=None, session=None, stored=None, stale_out=None,
1721
+ fields_session=None, vocabulary_memo=None):
1722
  """Validate a whole `perms` block. FAIL-CLOSED, and LOUD rather than lenient.
1723
 
1724
+ `fields_session` and `vocabulary_memo` are optional: `put_perms` passes the snapshot and memo
1725
+ `get_perms` builds, so the field vocabulary reads each store document once per save.
1726
+
1727
  ⭐⭐ W46-T35 — `stored` AND `stale_out` ARE OPTIONAL AND THE DEFAULT IS THE OLD BEHAVIOUR.
1728
  `stored` is the block this account ALREADY holds, handed down by `put_perms` from the
1729
  registry read it has already made — not a second read at a second moment, which is what the
 
1811
  # measured, not feared. `clean_filter_tree` takes bare strings and cannot see the flag,
1812
  # so the narrowing has to happen HERE, at the call.
1813
  # Reuse submitted verdicts in the picker vocabulary so each leaf is judged once.
1814
+ fields_here = _module_fields(key, session=fields_session or session,
1815
+ wall_verdicts=wall_verdicts,
1816
+ vocabulary_memo=vocabulary_memo)
1817
  valid_keys = {f["key"] for f in fields_here}
1818
  filter_keys = {f["key"] for f in fields_here if f.get("filterable")}
1819
  # ⭐⭐ W41-T09 / R8 — THE COLUMNS THE ADMIN-OWNER RULE TOOK OUT OF THE BUILDER, READ OFF
 
2246
  return {"conj": tree.get("conj", "and"), "nodes": nodes} if nodes else None
2247
 
2248
 
2249
+ def _perms_read_snapshot(session, roster):
2250
+ """A request-only copy of `session` that reads definitions, grants and the roster once.
2251
+
2252
+ The editor's picker and the save's validator both build their field vocabulary through it.
2253
+ It never enters a write door or outlives the request.
2254
+ """
2255
+ from copy import copy
2256
+ from core import user_tables
2257
+ from routes_customers import _AssemblyStore
2258
+
2259
+ snapshot = copy(session)
2260
+ snapshot._permission_roster = roster
2261
+ try:
2262
+ snapshot.runtime = user_tables.lend_defs(_AssemblyStore(session.runtime, schema_only=True))
2263
+ except Exception:
2264
+ raise err(503, "schema_unavailable", "The permission field definitions are unavailable. Try again.")
2265
+ return snapshot
2266
+
2267
+
2268
  @router.get("/admin/users/{username}/perms")
2269
  def get_perms(username: str, session: Session = Depends(admin_gate)):
2270
  """This account's permission block plus the field schema the editor needs to render it."""
 
2275
  # 404, never 403, because "that account exists in another company" is itself a leak.
2276
  raise err(404, "no_such_user", f"no account named {uname!r}")
2277
  import core.perm_scope as perm_scope
2278
+ from core import shares, freeze_notice
 
 
2279
 
2280
  try:
2281
  shares.reconcile_database_grants(session.runtime, tenant=session.tenant)
2282
  except freeze_notice.DataInputFrozen:
2283
  raise
2284
  except Exception as exc:
2285
+ # Boot retries the repair. The page still opens and names each tick left without a grant.
2286
+ import logging
2287
+ logging.getLogger(__name__).warning("Database grant repair deferred: %s", type(exc).__name__)
2288
 
2289
+ # Global account writes and their required post-write readback below use the original store.
2290
+ session = _perms_read_snapshot(session, reg)
 
 
 
 
 
 
 
2291
 
2292
  rec = reg[uname]
2293
  stored = rec.get("perms") or {}
 
2360
  if _gone:
2361
  stale_hidden_by_module[k] = _gone
2362
  perms_out = {}
2363
+ unsynced_access = []
2364
  for k in governed:
2365
  e = stored.get(k)
2366
  # ⛔⛔ W36-T22 — THE DEFAULT IS `may_read`, NOT `may_access`, AND THE DIFFERENCE IS AN
 
2395
  perm_scope.may_read(users._public(uname, rec), k, st=session.runtime))
2396
  if (str(k).startswith("ut_") and isinstance(e, dict)
2397
  and e.get("access") is True and not _default):
2398
+ # A tick whose share grant is missing still opens, so an admin can see it and re-save.
2399
+ unsynced_access.append(k)
2400
  # ⭐⭐ W38-T19 — `metrics` RIDES EVERY ENTRY, BACKFILLED TO GRANTED ON A RECORD WRITTEN
2401
  # BEFORE THE KEY EXISTED. The wire is TOTAL on purpose: the client's own parse already
2402
  # reads an absent key as granted, so this line changes no rendering — what it changes is
 
2444
  # `is_migrated` slip away from consuming `_fold_legacy_scope`'s one chance, which is
2445
  # the BU leak the guard above exists for.
2446
  "stale_hidden_by_module": stale_hidden_by_module,
2447
+ # The `ut_*` keys stored as access while the account cannot open them: a save repairs them.
2448
+ "unsynced_access": unsynced_access,
2449
  # ⭐⭐ W46-T34 — `{module: {count, refused, status, reason}}`, the metric census
2450
  # beside the field list it summarises. `status` is `ok` | `none` | `unavailable`,
2451
  # and the third is the one that had no way to be said before: an empty metric list
 
2541
  if not report["reconciliation"]["ok"]]}
2542
 
2543
 
2544
+ # The share sync writes one grant per changed database, so a save starts writing only with this left.
2545
+ _PERMS_WRITE_RESERVE_S = 20.0
2546
+
2547
+
2548
+ def _require_write_budget():
2549
+ """Refuse before the first write when the request deadline could fall between the two halves."""
2550
+ from core import request_timing
2551
+ try:
2552
+ left = request_timing.remaining_budget()
2553
+ except Exception:
2554
+ left = 0.0
2555
+ if left is not None and left < _PERMS_WRITE_RESERVE_S:
2556
+ raise err(503, "server_busy", "Access was not saved. Try again in a moment.")
2557
+
2558
+
2559
+ def _restore_perms(uname, prior, written):
2560
+ """Put back the perms block a failed share sync left, unless another save replaced it since."""
2561
+ from core import account_rows
2562
+
2563
+ def _set(reg):
2564
+ current = reg.get(uname)
2565
+ if isinstance(current, dict) and current.get("perms") == written.get("perms"):
2566
+ for field in ("perms", "perms_v"):
2567
+ if field in prior:
2568
+ current[field] = deepcopy(prior[field])
2569
+ else:
2570
+ current.pop(field, None)
2571
+ return reg
2572
+
2573
+ try:
2574
+ account_rows.update(uname, _set, st=users.store)
2575
+ except Exception as exc:
2576
+ import logging
2577
+ logging.getLogger(__name__).warning("Access restore after a failed share sync failed: %s",
2578
+ type(exc).__name__)
2579
+
2580
+
2581
+ def _sync_database_shares(session, uname, before, after, applied=None):
2582
  """R15/C5: explicit ut_* access grants editor authority; explicit off removes the share.
2583
 
2584
+ `applied`, when given, collects `(key, entries before, entries stored)` per write, so a
2585
+ caller whose save fails later can revert exactly these grants.
2586
+
2587
  Two doors answer two questions: the access rule decides WHICH rows, a `core.shares` `database`
2588
  grant decides IF the member sees and edits it, so the editor's toggle writes both. Re-saving
2589
  an enabled entry repairs old view-only grants. Other principals, owners and provenance stay
 
2623
  if now:
2624
  entries.append({**(held[0] if held else {}), "user": uname, "role": "edit"})
2625
  try:
2626
+ stored = shares.set_grants("database", key, entries, st=session.runtime,
2627
+ granter=session.uname, expect=record["entries"])
2628
  except shares.GrantsChanged as exc:
2629
  raise err(409, "shares_changed", "Database sharing changed. Save access again.") from exc
2630
+ if applied is not None:
2631
+ applied.append((key, record["entries"], stored["entries"]))
2632
+
2633
+
2634
+ def _revert_shares(session, applied):
2635
+ """Undo the grants a failed save wrote, newest first, each only while it still holds them."""
2636
+ from core import shares
2637
+
2638
+ for key, before, written in reversed(applied):
2639
+ try:
2640
+ shares.set_grants("database", key, before, st=session.runtime, expect=written)
2641
+ except Exception as exc:
2642
+ import logging
2643
+ logging.getLogger(__name__).warning("Grant revert after a failed save failed: %s",
2644
+ type(exc).__name__)
2645
 
2646
 
2647
  def _set_database_request_share(session, key, entries, held, snapshot):
 
2912
  # control for it: a report that lives only in a client this ticket does not own would be a
2913
  # feature nobody ships [[two-lanes-one-contract-dead-feature]].
2914
  _stale = {}
2915
+ prior = deepcopy(reg[uname])
2916
+ # Without the snapshot each database re-read the roster and schema: about 44 s on live, past
2917
+ # the 45 s request deadline, so the save failed after the perms block was written.
2918
  cleaned = _clean_perms(body.get("perms"),
2919
  governed_keys={m["key"] for m in _mods}, session=session,
2920
+ stored=reg[uname].get("perms"), stale_out=_stale,
2921
+ fields_session=_perms_read_snapshot(session, reg),
2922
+ vocabulary_memo={}) or {}
2923
  governed = {m["key"] for m in _mods}
2924
  opaque = {key: entry for key, entry in cleaned.items() if key not in governed}
2925
  known = _fold_legacy_scope(reg[uname],
 
2927
  session=session)
2928
  _guard_bu_widening(reg[uname], known, bool(body.get("confirm_widen")))
2929
  cleaned = dict(known, **opaque)
2930
+ _require_write_budget()
2931
  users.set_access(uname, perms=cleaned)
2932
  fresh = _registry()
2933
  rec = fresh.get(uname) or {}
 
2936
  # the wall is up when it is not — the one direction this must never fail in.
2937
  raise err(503, "store_unavailable",
2938
  "the permissions were not saved; the account is UNCHANGED")
2939
+ applied = []
2940
+ try:
2941
+ _sync_database_shares(session, uname, prior.get("perms") or {},
2942
+ {key: entry for key, entry in (rec.get("perms") or {}).items()
2943
+ if key in governed}, applied=applied)
2944
+ except Exception:
2945
+ # Both halves land or neither: undo this save's grants, then put the old block back. If
2946
+ # those writes fail too, the grant repair in `get_perms` and at boot follows the ticks.
2947
+ _revert_shares(session, applied)
2948
+ _restore_perms(uname, prior, rec)
2949
+ raise
2950
  return {"username": uname, "perms": rec.get("perms") or {},
2951
  "perms_v": int(rec.get("perms_v") or 0),
2952
  # W46-T35 — `{module: [key, ...]}`, empty on an ordinary save. What the save cleared
api/routes_automation.py CHANGED
@@ -2863,17 +2863,36 @@ if engine.grid_hook not in _ut_hooks.ROW_HOOKS:
2863
  _ut_hooks.ROW_HOOKS.append(engine.grid_hook)
2864
 
2865
 
 
 
 
2866
  def _received_projection(connector):
2867
  """Reuse each connected grid's canonical identities and field projection at publication."""
2868
  def resolve(rt, cursor):
 
 
2869
  from core import user_tables
2870
  import routes_odoo_tables
2871
  result = {}
 
 
 
 
 
2872
  prefix = 'ut_amzn_' if connector == 'amazon' else 'ut_odoo_'
2873
  for key in user_tables.all_defs(st=rt, schema_only=True):
2874
  if not str(key).startswith(prefix):
2875
  continue
2876
- spec = routes_odoo_tables._source_for(cursor, key, allow_lookups=False)
 
 
 
 
 
 
 
 
 
2877
  if spec and not spec.get('missing'):
2878
  result[key] = spec
2879
  return result
 
2863
  _ut_hooks.ROW_HOOKS.append(engine.grid_hook)
2864
 
2865
 
2866
+ _RECEIVED_NOT_READY = frozenset({'amazon_mirror_not_loaded', 'store_not_ready'})
2867
+
2868
+
2869
  def _received_projection(connector):
2870
  """Reuse each connected grid's canonical identities and field projection at publication."""
2871
  def resolve(rt, cursor):
2872
+ import logging
2873
+ from fastapi import HTTPException
2874
  from core import user_tables
2875
  import routes_odoo_tables
2876
  result = {}
2877
+ if connector == 'amazon':
2878
+ # Amazon specs raise BinderException on a publication's raw views and cost about 7 s
2879
+ # a table in the worker hand-off (sdlc85:88), so the projection waits for that fix.
2880
+ logging.getLogger(__name__).debug('received-change projection skips amazon until its specs bind')
2881
+ return result
2882
  prefix = 'ut_amzn_' if connector == 'amazon' else 'ut_odoo_'
2883
  for key in user_tables.all_defs(st=rt, schema_only=True):
2884
  if not str(key).startswith(prefix):
2885
  continue
2886
+ try:
2887
+ spec = routes_odoo_tables._source_for(cursor, key, allow_lookups=False)
2888
+ except HTTPException as exc:
2889
+ # A table still loading is skipped like a missing one, so it cannot drop its siblings.
2890
+ detail = exc.detail if isinstance(exc.detail, dict) else {}
2891
+ code = (detail.get('error') or {}).get('code')
2892
+ if code not in _RECEIVED_NOT_READY:
2893
+ raise
2894
+ logging.getLogger(__name__).debug('received-change projection skips %s (%s)', key, code)
2895
+ continue
2896
  if spec and not spec.get('missing'):
2897
  result[key] = spec
2898
  return result
api/routes_connectors.py CHANGED
@@ -3167,6 +3167,13 @@ def _amazon_cause_text(text, limit=300):
3167
  return text
3168
 
3169
 
 
 
 
 
 
 
 
3170
  def amazon_hold_cause(row=None, held="", *, table=""):
3171
  """One plain cause for an Amazon table a pass did not finish, from its report row or held text."""
3172
  from harness.amazon_store import plain_refusal_reason
@@ -3187,6 +3194,10 @@ def amazon_hold_cause(row=None, held="", *, table=""):
3187
  cause = "Amazon is still preparing this report."
3188
  elif state == "deferred" or (not row and (not held or held.startswith("not reached"))):
3189
  cause = "This pass ended before this table's turn. The next pass continues it."
 
 
 
 
3190
  elif state == "never_run":
3191
  # A table with no tick is due only in the minute after its daily slot, which a busy pass misses.
3192
  cause = "This table has never synced. Its daily start window is too short."
@@ -3306,7 +3317,8 @@ def _amazon_table_schedule(rt, connection_id=None, now=None):
3306
  reason = row_reason(table, state)
3307
  # A source-internal mismatch published the table. Its warning stays until a clean reconcile.
3308
  warning = _amazon_cause_text(saved.get(prefix + az.entity_key("warning_" + table)) or "")
3309
- schedule.append({**row, "connectionId": cid, "account": entry["label"],
 
3310
  "lastSyncAt": (completed.get("updated_at")
3311
  if completed.get("state") == "done" else None),
3312
  "state": state, **({"reason": reason} if reason else {}),
 
3167
  return text
3168
 
3169
 
3170
+ def _amazon_table_period(table):
3171
+ """The table's refresh period from the client CADENCE: "day", "week" or "month"."""
3172
+ from harness import amazon_store as az
3173
+ period = (az._client().CADENCE.get(str(table or "")) or {}).get("period")
3174
+ return period if period in ("day", "week", "month") else "day"
3175
+
3176
+
3177
  def amazon_hold_cause(row=None, held="", *, table=""):
3178
  """One plain cause for an Amazon table a pass did not finish, from its report row or held text."""
3179
  from harness.amazon_store import plain_refusal_reason
 
3194
  cause = "Amazon is still preparing this report."
3195
  elif state == "deferred" or (not row and (not held or held.startswith("not reached"))):
3196
  cause = "This pass ended before this table's turn. The next pass continues it."
3197
+ elif state == "never_run" and _amazon_table_period(table) in ("week", "month"):
3198
+ # A weekly or monthly table waits for its first calendar period, so a missing tick is expected.
3199
+ cause = (f"This table has not synced yet. Its first "
3200
+ f"{'weekly' if _amazon_table_period(table) == 'week' else 'monthly'} pass is scheduled.")
3201
  elif state == "never_run":
3202
  # A table with no tick is due only in the minute after its daily slot, which a busy pass misses.
3203
  cause = "This table has never synced. Its daily start window is too short."
 
3317
  reason = row_reason(table, state)
3318
  # A source-internal mismatch published the table. Its warning stays until a clean reconcile.
3319
  warning = _amazon_cause_text(saved.get(prefix + az.entity_key("warning_" + table)) or "")
3320
+ schedule.append({**row, "period": _amazon_table_period(table),
3321
+ "connectionId": cid, "account": entry["label"],
3322
  "lastSyncAt": (completed.get("updated_at")
3323
  if completed.get("state") == "done" else None),
3324
  "state": state, **({"reason": reason} if reason else {}),
platform/core/connector_spec.py CHANGED
@@ -815,7 +815,7 @@ AMAZON_TABLES = (
815
  ConnectorTableSpec(
816
  key=_AMZN + "review_topics", native="customerFeedback/2024-06-01/items/{asin}/reviews/topics",
817
  source="amzn_review_topics", label="Amazon Listings: Review topics", pinned="asin",
818
- identity=("asin", "marketplace_id", "sentiment", "topic"),
819
  links=(("asin", _AMZN + "listings", "asin1", "many_to_one"),),
820
  ),
821
  ConnectorTableSpec(
@@ -824,6 +824,13 @@ AMAZON_TABLES = (
824
  identity=("asin", "marketplace_id", "sentiment", "topic", "month"),
825
  links=(("asin", _AMZN + "listings", "asin1", "many_to_one"),),
826
  ),
 
 
 
 
 
 
 
827
  ConnectorTableSpec(
828
  key=_AMZN + "sku_keywords", native="derived:listing_items+search_query_performance",
829
  source="amzn_sku_keywords", label="Amazon Marketing: SKU keywords", pinned="sku",
@@ -3306,6 +3313,39 @@ AMAZON_LABELS = {
3306
  "review_snippet_3": "Review snippet 3",
3307
  "review_snippets": "Review snippets",
3308
  "subtopics": "Subtopics",
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3309
  },
3310
  _AMZN + "review_trends": {
3311
  "_data_origin": "Data origin",
@@ -3321,6 +3361,57 @@ AMAZON_LABELS = {
3321
  "date_range_start_date": "Review period start",
3322
  "date_range_end_date": "Review period end",
3323
  "asin_metrics_occurrence_percentage": "ASIN occurrence",
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3324
  },
3325
  _AMZN + "replenishment": {
3326
  "_data_origin": "Data origin",
@@ -3491,12 +3582,15 @@ AMAZON_DEFAULT_COLUMNS = {
3491
  _AMZN + "replenishment": ("asin", "time_interval_start_date", "time_interval_end_date",
3492
  "active_subscriptions", "shipped_subscription_units", "total_subscriptions_revenue",
3493
  "revenue_penetration", "subscriber_retention_for30_days", "program_type"),
3494
- _AMZN + "review_topics": ("sentiment", "topic", "asin_metrics_number_of_mentions",
3495
  "asin_metrics_occurrence_percentage", "asin_metrics_star_rating_impact",
3496
  "parent_asin_metrics_number_of_mentions", "browse_node_metrics_occurrence_percentage_all_products",
3497
  "review_snippet_1", "review_snippet_2", "review_snippet_3"),
3498
  _AMZN + "review_trends": ("sentiment", "topic", "month", "asin_metrics_occurrence_percentage",
3499
  "date_range_start_date", "date_range_end_date"),
 
 
 
3500
  _AMZN + "sku_keywords": ("sku", "asin", "month", "search_query", "impressions", "clicks",
3501
  "cart_adds", "purchases", "backend_keywords", "source_basis"),
3502
  _AMZN + "keyword_ranks": ("keyword", "date", "organic_rank", "sponsored_rank",
@@ -3623,7 +3717,8 @@ AMAZON_GROUPS = {
3623
  **{_AMZN + t: "Listings" for t in (
3624
  "listings", "pricing", "inactive_listings", "cancelled_listings", "suppressed_listings",
3625
  "listing_defects", "sold_listings", "listing_items", "catalog_items", "b2b_recommended",
3626
- "b2b_not_yet_on_amazon", "geo_opportunities", "b2b_quotes", "review_topics", "review_trends")},
 
3627
  **{_AMZN + t: "Advertising" for t in (
3628
  "brand_analytics", "search_query_performance", "search_catalog_performance",
3629
  "market_basket", "repeat_purchase", "promotion_performance", "coupon_performance",
 
815
  ConnectorTableSpec(
816
  key=_AMZN + "review_topics", native="customerFeedback/2024-06-01/items/{asin}/reviews/topics",
817
  source="amzn_review_topics", label="Amazon Listings: Review topics", pinned="asin",
818
+ identity=("asin", "marketplace_id", "sort_by", "sentiment", "topic"),
819
  links=(("asin", _AMZN + "listings", "asin1", "many_to_one"),),
820
  ),
821
  ConnectorTableSpec(
 
824
  identity=("asin", "marketplace_id", "sentiment", "topic", "month"),
825
  links=(("asin", _AMZN + "listings", "asin1", "many_to_one"),),
826
  ),
827
+ ConnectorTableSpec(
828
+ key=_AMZN + "browse_node_review_topics",
829
+ native="customerFeedback/2024-06-01/browseNodes/{browseNodeId}/reviews/topics",
830
+ source="amzn_browse_node_review_topics", label="Amazon Listings: Category review topics",
831
+ pinned="browse_node_name",
832
+ identity=("browse_node_id", "marketplace_id", "sort_by", "sentiment", "topic"),
833
+ ),
834
  ConnectorTableSpec(
835
  key=_AMZN + "sku_keywords", native="derived:listing_items+search_query_performance",
836
  source="amzn_sku_keywords", label="Amazon Marketing: SKU keywords", pinned="sku",
 
3313
  "review_snippet_3": "Review snippet 3",
3314
  "review_snippets": "Review snippets",
3315
  "subtopics": "Subtopics",
3316
+ "sort_by": "Sorted by",
3317
+ "browse_node_id": "Browse node ID",
3318
+ "browse_node_name": "Category",
3319
+ "subtopic_1": "Subtopic 1",
3320
+ "subtopic_1_number_of_mentions": "Subtopic 1 mentions",
3321
+ "subtopic_1_occurrence_percentage": "Subtopic 1 occurrence",
3322
+ "subtopic_1_review_snippet_1": "First subtopic snippet 1",
3323
+ "subtopic_1_review_snippet_2": "First subtopic snippet 2",
3324
+ "subtopic_1_review_snippet_3": "First subtopic snippet 3",
3325
+ "subtopic_2": "Subtopic 2",
3326
+ "subtopic_2_number_of_mentions": "Subtopic 2 mentions",
3327
+ "subtopic_2_occurrence_percentage": "Subtopic 2 occurrence",
3328
+ "subtopic_2_review_snippet_1": "Second subtopic snippet 1",
3329
+ "subtopic_2_review_snippet_2": "Second subtopic snippet 2",
3330
+ "subtopic_2_review_snippet_3": "Second subtopic snippet 3",
3331
+ "subtopic_3": "Subtopic 3",
3332
+ "subtopic_3_number_of_mentions": "Subtopic 3 mentions",
3333
+ "subtopic_3_occurrence_percentage": "Subtopic 3 occurrence",
3334
+ "subtopic_3_review_snippet_1": "Third subtopic snippet 1",
3335
+ "subtopic_3_review_snippet_2": "Third subtopic snippet 2",
3336
+ "subtopic_3_review_snippet_3": "Third subtopic snippet 3",
3337
+ "subtopic_4": "Subtopic 4",
3338
+ "subtopic_4_number_of_mentions": "Subtopic 4 mentions",
3339
+ "subtopic_4_occurrence_percentage": "Subtopic 4 occurrence",
3340
+ "subtopic_4_review_snippet_1": "Fourth subtopic snippet 1",
3341
+ "subtopic_4_review_snippet_2": "Fourth subtopic snippet 2",
3342
+ "subtopic_4_review_snippet_3": "Fourth subtopic snippet 3",
3343
+ "subtopic_5": "Subtopic 5",
3344
+ "subtopic_5_number_of_mentions": "Subtopic 5 mentions",
3345
+ "subtopic_5_occurrence_percentage": "Subtopic 5 occurrence",
3346
+ "subtopic_5_review_snippet_1": "Fifth subtopic snippet 1",
3347
+ "subtopic_5_review_snippet_2": "Fifth subtopic snippet 2",
3348
+ "subtopic_5_review_snippet_3": "Fifth subtopic snippet 3",
3349
  },
3350
  _AMZN + "review_trends": {
3351
  "_data_origin": "Data origin",
 
3361
  "date_range_start_date": "Review period start",
3362
  "date_range_end_date": "Review period end",
3363
  "asin_metrics_occurrence_percentage": "ASIN occurrence",
3364
+ "parent_asin_metrics_occurrence_percentage": "Parent ASIN occurrence",
3365
+ "browse_node_metrics_occurrence_percentage_all_products": "Browse node occurrence",
3366
+ },
3367
+ _AMZN + "browse_node_review_topics": {
3368
+ "_data_origin": "Data origin",
3369
+ "_report_document_id": "Report document ID",
3370
+ "_report_row_number": "Report row",
3371
+ "_report_type": "Report type",
3372
+ "_snapshot_date": "Synced on",
3373
+ "browse_node_id": "Browse node ID",
3374
+ "browse_node_name": "Category",
3375
+ "marketplace_id": "Marketplace ID",
3376
+ "sort_by": "Sorted by",
3377
+ "sentiment": "Sentiment",
3378
+ "topic": "Topic",
3379
+ "date_range_start_date": "Review period start",
3380
+ "date_range_end_date": "Review period end",
3381
+ "browse_node_metrics_occurrence_percentage_all_products": "Category occurrence",
3382
+ "browse_node_metrics_occurrence_percentage_top_twenty_five_percent_products": "Top 25 percent occurrence",
3383
+ "browse_node_metrics_star_rating_impact_all_products": "Category star rating impact",
3384
+ "browse_node_metrics_star_rating_impact_top_twenty_five_percent_products": "Top 25 percent star rating impact",
3385
+ "review_snippet_1": "Review snippet 1",
3386
+ "review_snippet_2": "Review snippet 2",
3387
+ "review_snippet_3": "Review snippet 3",
3388
+ "subtopic_1": "Subtopic 1",
3389
+ "subtopic_1_occurrence_percentage": "Subtopic 1 occurrence",
3390
+ "subtopic_1_review_snippet_1": "First subtopic snippet 1",
3391
+ "subtopic_1_review_snippet_2": "First subtopic snippet 2",
3392
+ "subtopic_1_review_snippet_3": "First subtopic snippet 3",
3393
+ "subtopic_2": "Subtopic 2",
3394
+ "subtopic_2_occurrence_percentage": "Subtopic 2 occurrence",
3395
+ "subtopic_2_review_snippet_1": "Second subtopic snippet 1",
3396
+ "subtopic_2_review_snippet_2": "Second subtopic snippet 2",
3397
+ "subtopic_2_review_snippet_3": "Second subtopic snippet 3",
3398
+ "subtopic_3": "Subtopic 3",
3399
+ "subtopic_3_occurrence_percentage": "Subtopic 3 occurrence",
3400
+ "subtopic_3_review_snippet_1": "Third subtopic snippet 1",
3401
+ "subtopic_3_review_snippet_2": "Third subtopic snippet 2",
3402
+ "subtopic_3_review_snippet_3": "Third subtopic snippet 3",
3403
+ "subtopic_4": "Subtopic 4",
3404
+ "subtopic_4_occurrence_percentage": "Subtopic 4 occurrence",
3405
+ "subtopic_4_review_snippet_1": "Fourth subtopic snippet 1",
3406
+ "subtopic_4_review_snippet_2": "Fourth subtopic snippet 2",
3407
+ "subtopic_4_review_snippet_3": "Fourth subtopic snippet 3",
3408
+ "subtopic_5": "Subtopic 5",
3409
+ "subtopic_5_occurrence_percentage": "Subtopic 5 occurrence",
3410
+ "subtopic_5_review_snippet_1": "Fifth subtopic snippet 1",
3411
+ "subtopic_5_review_snippet_2": "Fifth subtopic snippet 2",
3412
+ "subtopic_5_review_snippet_3": "Fifth subtopic snippet 3",
3413
+ "our_asins": "Our ASINs",
3414
+ "our_asin_count": "Our ASIN count",
3415
  },
3416
  _AMZN + "replenishment": {
3417
  "_data_origin": "Data origin",
 
3582
  _AMZN + "replenishment": ("asin", "time_interval_start_date", "time_interval_end_date",
3583
  "active_subscriptions", "shipped_subscription_units", "total_subscriptions_revenue",
3584
  "revenue_penetration", "subscriber_retention_for30_days", "program_type"),
3585
+ _AMZN + "review_topics": ("sort_by", "sentiment", "topic", "browse_node_name", "asin_metrics_number_of_mentions",
3586
  "asin_metrics_occurrence_percentage", "asin_metrics_star_rating_impact",
3587
  "parent_asin_metrics_number_of_mentions", "browse_node_metrics_occurrence_percentage_all_products",
3588
  "review_snippet_1", "review_snippet_2", "review_snippet_3"),
3589
  _AMZN + "review_trends": ("sentiment", "topic", "month", "asin_metrics_occurrence_percentage",
3590
  "date_range_start_date", "date_range_end_date"),
3591
+ _AMZN + "browse_node_review_topics": ("sort_by", "sentiment", "topic",
3592
+ "browse_node_metrics_occurrence_percentage_all_products",
3593
+ "browse_node_metrics_star_rating_impact_all_products", "our_asin_count", "review_snippet_1"),
3594
  _AMZN + "sku_keywords": ("sku", "asin", "month", "search_query", "impressions", "clicks",
3595
  "cart_adds", "purchases", "backend_keywords", "source_basis"),
3596
  _AMZN + "keyword_ranks": ("keyword", "date", "organic_rank", "sponsored_rank",
 
3717
  **{_AMZN + t: "Listings" for t in (
3718
  "listings", "pricing", "inactive_listings", "cancelled_listings", "suppressed_listings",
3719
  "listing_defects", "sold_listings", "listing_items", "catalog_items", "b2b_recommended",
3720
+ "b2b_not_yet_on_amazon", "geo_opportunities", "b2b_quotes", "review_topics", "review_trends",
3721
+ "browse_node_review_topics")},
3722
  **{_AMZN + t: "Advertising" for t in (
3723
  "brand_analytics", "search_query_performance", "search_catalog_performance",
3724
  "market_basket", "repeat_purchase", "promotion_performance", "coupon_performance",
platform/core/shares.py CHANGED
@@ -598,7 +598,7 @@ def reconcile_database_grants(st=None, *, tenant=None, table_keys=None):
598
  def _pending(data):
599
  if not isinstance(data, dict) or not isinstance(data.get('database', {}), dict):
600
  raise RuntimeError('database sharing is unavailable')
601
- registry = users.store._read_strict('users')
602
  if not isinstance(registry, dict):
603
  raise RuntimeError('the account registry is unavailable')
604
  database = data.get('database', {})
@@ -692,6 +692,18 @@ def reconcile_database_grants(st=None, *, tenant=None, table_keys=None):
692
  return repaired[0]
693
 
694
 
 
 
 
 
 
 
 
 
 
 
 
 
695
  def repair_published_database_grants(st=None, *, table_keys):
696
  """Attempt one repair after publication, deferring failures to boot/editor except freeze refusal."""
697
  from core import freeze_notice
 
598
  def _pending(data):
599
  if not isinstance(data, dict) or not isinstance(data.get('database', {}), dict):
600
  raise RuntimeError('database sharing is unavailable')
601
+ registry = _strict_accounts()
602
  if not isinstance(registry, dict):
603
  raise RuntimeError('the account registry is unavailable')
604
  database = data.get('database', {})
 
692
  return repaired[0]
693
 
694
 
695
+ def _strict_accounts():
696
+ """Every account record, read strictly: the account rows on PostgreSQL, else the users document."""
697
+ from core import account_rows, users
698
+ # On PostgreSQL the users document is a layout marker; reading it repairs no account at all.
699
+ if account_rows._pg_source(users.store):
700
+ return dict(account_rows._all(users.store))
701
+ registry = users.store._read_strict('users')
702
+ if account_rows.is_marker(registry):
703
+ raise RuntimeError('the account registry is unavailable')
704
+ return registry
705
+
706
+
707
  def repair_published_database_grants(st=None, *, table_keys):
708
  """Attempt one repair after publication, deferring failures to boot/editor except freeze refusal."""
709
  from core import freeze_notice
platform/core/store_pg.py CHANGED
@@ -2719,9 +2719,10 @@ def download_bytes(path_in_repo, tenant_slug=None):
2719
  """Raw bytes, or None when absent — the same contract as the HF backend: a missing attachment
2720
  degrades to "that file is gone", never to a broken page."""
2721
  with _pool().connection() as con:
 
2722
  row = con.execute(
2723
  _q('SELECT bytes FROM {tbl} WHERE path = %s', 'store_blobs', tenant_slug),
2724
- (str(path_in_repo),)).fetchone()
2725
  return bytes(row[0]) if row else None
2726
 
2727
 
 
2719
  """Raw bytes, or None when absent — the same contract as the HF backend: a missing attachment
2720
  degrades to "that file is gone", never to a broken page."""
2721
  with _pool().connection() as con:
2722
+ # Binary results: a text-format bytea arrives as hex and costs about 3x the transfer.
2723
  row = con.execute(
2724
  _q('SELECT bytes FROM {tbl} WHERE path = %s', 'store_blobs', tenant_slug),
2725
+ (str(path_in_repo),), binary=True).fetchone()
2726
  return bytes(row[0]) if row else None
2727
 
2728
 
platform/harness/amazon_client.py CHANGED
@@ -117,6 +117,8 @@ _entity_pacers = {
117
  "searchCatalogItems": {"last": None, "lock": threading.Lock()},
118
  "getItemReviewTopics": {"last": None, "lock": threading.Lock()},
119
  "getItemReviewTrends": {"last": None, "lock": threading.Lock()},
 
 
120
  "getMyFeesEstimates": {"last": None, "lock": threading.Lock()},
121
  "listAllFulfillmentOrders": {"last": None, "lock": threading.Lock()},
122
  "getFulfillmentOrder": {"last": None, "lock": threading.Lock()},
@@ -937,6 +939,16 @@ GROUP_A_TABLES = {
937
  "snapshot": True,
938
  "validates_against": "Amazon raw trend point counts and occurrence percentages per ASIN, sentiment, topic and month vs stored rows for the same pass",
939
  },
 
 
 
 
 
 
 
 
 
 
940
  }
941
  REPORT_TYPES.update(GROUP_A_TABLES)
942
 
@@ -1251,6 +1263,13 @@ OPERATIONS = {
1251
  "getItemReviewTrends": {"name": "getItemReviewTrends",
1252
  "roles": ("Brand Analytics", "Selling Partner Insights"), "role_match": "any",
1253
  "method": "GET", "path": "customerFeedback/2024-06-01/items/{asin}/reviews/trends"},
 
 
 
 
 
 
 
1254
  "listAllFulfillmentOrders": {"name": "listAllFulfillmentOrders", "roles": ("Amazon Fulfillment",),
1255
  "method": "GET", "path": "fulfillmentOutbound/2020-07-01/fulfillmentOrders"},
1256
  "getFulfillmentOrder": {"name": "getFulfillmentOrder", "roles": ("Amazon Fulfillment",),
@@ -1883,7 +1902,10 @@ def transactions(fields, start, end, *, timeout=_TIMEOUT, sleep=time.sleep, now=
1883
  ENTITY_INTERVALS = {"getOrderMetrics": 2.0, "getInventorySummaries": 0.5,
1884
  "searchListingsItems": 0.2, "searchCatalogItems": 0.5,
1885
  "getMyFeesEstimates": 2.0}
1886
- FEEDBACK_INTERVALS = {"getItemReviewTopics": 1.0, "getItemReviewTrends": 1.0}
 
 
 
1887
  INBOUND_INTERVALS = {"listInboundPlans": 0.5, "getInboundPlan": 0.5, "listShipmentItems": 0.5,
1888
  "getShipment": 0.5, "getShipmentItemsByShipmentId": 0.5, "getShipmentItems": 0.5}
1889
  #: Amazon's 400 text for an Amazon Warehousing and Distribution plan in the FBA inbound list.
@@ -2242,46 +2264,168 @@ def inbound_shipment_items(fields, *, timeout=_TIMEOUT, sleep=time.sleep, now=ti
2242
  return InboundRecords(records, skipped)
2243
 
2244
 
2245
- def item_review_feedback(fields, asins, *, operation="getItemReviewTopics", timeout=_TIMEOUT,
2246
- sleep=time.sleep, now=time.monotonic):
2247
- """Read every child ASIN, retaining known absences separately from invalid responses."""
2248
  if operation not in FEEDBACK_INTERVALS:
2249
  raise AmazonOperationRefused("unapproved Customer Feedback reader")
2250
  if not set(OPERATIONS[operation]["roles"]).intersection(ROLES_TAKEN):
2251
  raise AmazonOperationRefused("Customer Feedback needs Brand Analytics or Selling Partner Insights")
 
 
 
2252
  identifiers = sorted(set(asins))
2253
  if any(not isinstance(asin, str) or not re.fullmatch(r"[A-Z0-9]{10}", asin) for asin in identifiers):
2254
  raise AmazonError("Customer Feedback contains an invalid child ASIN. Correct the listings catalogue and retry.")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2255
  if not identifiers:
2256
  return []
2257
  marketplace = fields.get("marketplace_id") or US_MARKETPLACE_ID
2258
- params = {"marketplaceId": marketplace}
2259
- if operation == "getItemReviewTopics":
2260
- params["sortBy"] = "MENTIONS"
2261
- root = "topics" if operation == "getItemReviewTopics" else "reviewTrends"
2262
- token, issued = access_token(fields, timeout=timeout), now()
2263
- records = []
2264
- for asin in identifiers:
2265
- _pace_entity(operation, FEEDBACK_INTERVALS[operation], sleep=sleep, now=now)
2266
- if now() - issued >= 3000:
2267
- token, issued = access_token(fields, timeout=timeout), now()
2268
- body = _request(OPERATIONS[operation]["path"].replace("{asin}", asin), token,
2269
- region=str(fields.get("region") or DEFAULT_REGION), sandbox=is_sandbox(fields),
2270
- timeout=timeout, throttle_fix="continue every child ASIN at the Customer Feedback rate limit",
2271
- **params)
2272
- if body is None:
2273
- records.append({"asin": asin, "marketplaceId": marketplace, "_no_feedback_reason":
2274
- f"Amazon {operation} returned 204 No Content for this ASIN and marketplace."})
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2275
  continue
2276
- if (not isinstance(body, dict) or body.get("asin") != asin or body.get("marketplaceId") != marketplace
2277
- or not isinstance(body.get(root), dict)):
2278
- raise AmazonError(f"Amazon {operation} returned an invalid response for the requested child ASIN and marketplace. Retry after Amazon restores its response.")
2279
  for key in ("positiveTopics", "negativeTopics"):
2280
- topics = body[root].get(key)
2281
  if topics is not None and (not isinstance(topics, list) or any(not isinstance(t, dict) for t in topics)):
2282
- raise AmazonError(f"Amazon {operation} returned invalid {key}. No partial snapshot was stored.")
2283
- records.append(body)
2284
- return records
2285
 
2286
 
2287
  def group_a_records(fields, operation, *, start=None, end=None, asins=(),
@@ -2685,8 +2829,10 @@ def _request(path, token, region=DEFAULT_REGION, method="GET", timeout=_TIMEOUT,
2685
  if path.startswith("notifications/v1/"):
2686
  reason = (detail or "Amazon refused this notification operation.").replace(token, "[redacted]")
2687
  raise AmazonNotificationError(e.code, reason.replace("\u2014", ". ").replace("\u2013", " to ")) from None
2688
- raise AmazonError(f"{_safe(url)} answered {e.code}"
2689
- + (f": {detail}" if detail else "")) from None
 
 
2690
  except Exception as e: # noqa: BLE001
2691
  raise AmazonError(f"{_safe(url)} unreachable: {type(e).__name__}") from None
2692
 
 
117
  "searchCatalogItems": {"last": None, "lock": threading.Lock()},
118
  "getItemReviewTopics": {"last": None, "lock": threading.Lock()},
119
  "getItemReviewTrends": {"last": None, "lock": threading.Lock()},
120
+ "getItemBrowseNode": {"last": None, "lock": threading.Lock()},
121
+ "getBrowseNodeReviewTopics": {"last": None, "lock": threading.Lock()},
122
  "getMyFeesEstimates": {"last": None, "lock": threading.Lock()},
123
  "listAllFulfillmentOrders": {"last": None, "lock": threading.Lock()},
124
  "getFulfillmentOrder": {"last": None, "lock": threading.Lock()},
 
939
  "snapshot": True,
940
  "validates_against": "Amazon raw trend point counts and occurrence percentages per ASIN, sentiment, topic and month vs stored rows for the same pass",
941
  },
942
+ "browse_node_review_topics": {
943
+ "label": "Amazon category review topics",
944
+ "api": "getBrowseNodeReviewTopics",
945
+ "role": "Brand Analytics / Selling Partner Insights",
946
+ "role_match": "any",
947
+ "min_interval_s": 7 * 86400,
948
+ "cadence_note": "Amazon refreshes the top 10 positive and negative category topics weekly. Every browse node of every active child ASIN is requested, sorted by mentions and by star rating impact.",
949
+ "snapshot": True,
950
+ "validates_against": "Amazon raw positive and negative topic counts and metrics per browse node and sort vs stored topics for the same pass",
951
+ },
952
  }
953
  REPORT_TYPES.update(GROUP_A_TABLES)
954
 
 
1263
  "getItemReviewTrends": {"name": "getItemReviewTrends",
1264
  "roles": ("Brand Analytics", "Selling Partner Insights"), "role_match": "any",
1265
  "method": "GET", "path": "customerFeedback/2024-06-01/items/{asin}/reviews/trends"},
1266
+ "getItemBrowseNode": {"name": "getItemBrowseNode",
1267
+ "roles": ("Brand Analytics", "Selling Partner Insights"), "role_match": "any",
1268
+ "method": "GET", "path": "customerFeedback/2024-06-01/items/{asin}/browseNode"},
1269
+ "getBrowseNodeReviewTopics": {"name": "getBrowseNodeReviewTopics",
1270
+ "roles": ("Brand Analytics", "Selling Partner Insights"), "role_match": "any",
1271
+ "method": "GET",
1272
+ "path": "customerFeedback/2024-06-01/browseNodes/{browseNodeId}/reviews/topics"},
1273
  "listAllFulfillmentOrders": {"name": "listAllFulfillmentOrders", "roles": ("Amazon Fulfillment",),
1274
  "method": "GET", "path": "fulfillmentOutbound/2020-07-01/fulfillmentOrders"},
1275
  "getFulfillmentOrder": {"name": "getFulfillmentOrder", "roles": ("Amazon Fulfillment",),
 
1902
  ENTITY_INTERVALS = {"getOrderMetrics": 2.0, "getInventorySummaries": 0.5,
1903
  "searchListingsItems": 0.2, "searchCatalogItems": 0.5,
1904
  "getMyFeesEstimates": 2.0}
1905
+ FEEDBACK_INTERVALS = {"getItemReviewTopics": 1.0, "getItemReviewTrends": 1.0,
1906
+ "getItemBrowseNode": 1.0, "getBrowseNodeReviewTopics": 1.0}
1907
+ #: getItemReviewTopics and getBrowseNodeReviewTopics accept exactly these sortBy values.
1908
+ FEEDBACK_SORTS = ("MENTIONS", "STAR_RATING_IMPACT")
1909
  INBOUND_INTERVALS = {"listInboundPlans": 0.5, "getInboundPlan": 0.5, "listShipmentItems": 0.5,
1910
  "getShipment": 0.5, "getShipmentItemsByShipmentId": 0.5, "getShipmentItems": 0.5}
1911
  #: Amazon's 400 text for an Amazon Warehousing and Distribution plan in the FBA inbound list.
 
2264
  return InboundRecords(records, skipped)
2265
 
2266
 
2267
+ def _feedback_admitted(operation):
2268
+ """Refuse an unapproved Customer Feedback operation or a missing role before any transport."""
 
2269
  if operation not in FEEDBACK_INTERVALS:
2270
  raise AmazonOperationRefused("unapproved Customer Feedback reader")
2271
  if not set(OPERATIONS[operation]["roles"]).intersection(ROLES_TAKEN):
2272
  raise AmazonOperationRefused("Customer Feedback needs Brand Analytics or Selling Partner Insights")
2273
+
2274
+
2275
+ def _feedback_child_asins(asins):
2276
  identifiers = sorted(set(asins))
2277
  if any(not isinstance(asin, str) or not re.fullmatch(r"[A-Z0-9]{10}", asin) for asin in identifiers):
2278
  raise AmazonError("Customer Feedback contains an invalid child ASIN. Correct the listings catalogue and retry.")
2279
+ return identifiers
2280
+
2281
+
2282
+ def _feedback_sort(sort_by):
2283
+ if sort_by not in FEEDBACK_SORTS:
2284
+ raise AmazonError("Customer Feedback sortBy must be MENTIONS or STAR_RATING_IMPACT")
2285
+ return sort_by
2286
+
2287
+
2288
+ def _feedback_reads(fields, keys, reads, *, subject, timeout, sleep, now):
2289
+ """Yield `(key, label, body, reason)` per key and read, interleaved so each operation paces its own bucket.
2290
+
2291
+ A 204, 400 or 404 answers one key: body None and a reason. A 403, an exhausted 429 or any other refusal raises.
2292
+ """
2293
+ token, issued = access_token(fields, timeout=timeout), now()
2294
+ for key in keys:
2295
+ for label, operation, path, params in reads(key):
2296
+ _pace_entity(operation, FEEDBACK_INTERVALS[operation], sleep=sleep, now=now)
2297
+ if now() - issued >= 3000:
2298
+ token, issued = access_token(fields, timeout=timeout), now()
2299
+ try:
2300
+ body = _request(path, token, region=str(fields.get("region") or DEFAULT_REGION),
2301
+ sandbox=is_sandbox(fields), timeout=timeout,
2302
+ throttle_fix="continue every child ASIN at the Customer Feedback rate limit",
2303
+ **params)
2304
+ except AmazonError as exc:
2305
+ status = getattr(exc, "status", None)
2306
+ if isinstance(exc, (AmazonThrottled, AmazonOperationRefused)) or status not in (400, 404):
2307
+ raise
2308
+ detail = str(getattr(exc, "vendor_message", "") or "").replace("\u2014", ". ").replace("\u2013", " to ")
2309
+ detail = re.sub(r"[.\s]+$", "", detail.strip())
2310
+ yield key, label, None, (f"Amazon {operation} answered {status} for this {subject} and marketplace: "
2311
+ + (detail or "no reason given") + ".")
2312
+ continue
2313
+ if body is None:
2314
+ yield key, label, None, f"Amazon {operation} returned 204 No Content for this {subject} and marketplace."
2315
+ continue
2316
+ yield key, label, body, None
2317
+
2318
+
2319
+ def _feedback_record(operation, asin, marketplace, body, reason):
2320
+ """One validated item response, or its absence record."""
2321
+ if reason is not None:
2322
+ return {"asin": asin, "marketplaceId": marketplace, "_no_feedback_reason": reason}
2323
+ root = "topics" if operation == "getItemReviewTopics" else "reviewTrends"
2324
+ if (not isinstance(body, dict) or body.get("asin") != asin or body.get("marketplaceId") != marketplace
2325
+ or not isinstance(body.get(root), dict)):
2326
+ raise AmazonError(f"Amazon {operation} returned an invalid response for the requested child ASIN and marketplace. Retry after Amazon restores its response.")
2327
+ for key in ("positiveTopics", "negativeTopics"):
2328
+ topics = body[root].get(key)
2329
+ if topics is not None and (not isinstance(topics, list) or any(not isinstance(t, dict) for t in topics)):
2330
+ raise AmazonError(f"Amazon {operation} returned invalid {key}. No partial snapshot was stored.")
2331
+ return body
2332
+
2333
+
2334
+ def _browse_node_record(body, reason):
2335
+ """`{"browse_node_id", "browse_node_name"}` from getItemBrowseNode, or its absence."""
2336
+ if reason is not None:
2337
+ return {"_no_feedback_reason": reason}
2338
+ node, name = (body.get(k) if isinstance(body, dict) else None for k in ("browseNodeId", "displayName"))
2339
+ if not isinstance(node, str) or not re.fullmatch(r"\d{1,20}", node) or (name is not None and not isinstance(name, str)):
2340
+ raise AmazonError("Amazon getItemBrowseNode returned no valid browse node. No partial snapshot was stored.")
2341
+ return {"browse_node_id": node, "browse_node_name": name}
2342
+
2343
+
2344
+ def item_review_feedback(fields, asins, *, operation="getItemReviewTopics", sort_by="MENTIONS", timeout=_TIMEOUT,
2345
+ sleep=time.sleep, now=time.monotonic):
2346
+ """Read every child ASIN, retaining known absences separately from invalid responses."""
2347
+ if operation not in ("getItemReviewTopics", "getItemReviewTrends"):
2348
+ raise AmazonOperationRefused("unapproved Customer Feedback reader")
2349
+ _feedback_admitted(operation)
2350
+ params = {"sortBy": _feedback_sort(sort_by)} if operation == "getItemReviewTopics" else {}
2351
+ identifiers = _feedback_child_asins(asins)
2352
  if not identifiers:
2353
  return []
2354
  marketplace = fields.get("marketplace_id") or US_MARKETPLACE_ID
2355
+ path = OPERATIONS[operation]["path"]
2356
+ reads = lambda asin: [(None, operation, path.replace("{asin}", asin), {"marketplaceId": marketplace, **params})]
2357
+ return [_feedback_record(operation, asin, marketplace, body, reason) for asin, _label, body, reason in
2358
+ _feedback_reads(fields, identifiers, reads, subject="ASIN", timeout=timeout, sleep=sleep, now=now)]
2359
+
2360
+
2361
+ def item_browse_nodes(fields, asins, *, timeout=_TIMEOUT, sleep=time.sleep, now=time.monotonic):
2362
+ """`{asin: {"browse_node_id", "browse_node_name"}}` for every child ASIN; an absence carries `_no_feedback_reason`."""
2363
+ _feedback_admitted("getItemBrowseNode")
2364
+ identifiers = _feedback_child_asins(asins)
2365
+ if not identifiers:
2366
+ return {}
2367
+ marketplace = fields.get("marketplace_id") or US_MARKETPLACE_ID
2368
+ path = OPERATIONS["getItemBrowseNode"]["path"]
2369
+ reads = lambda asin: [(None, "getItemBrowseNode", path.replace("{asin}", asin), {"marketplaceId": marketplace})]
2370
+ return {asin: _browse_node_record(body, reason) for asin, _label, body, reason in
2371
+ _feedback_reads(fields, identifiers, reads, subject="ASIN", timeout=timeout, sleep=sleep, now=now)}
2372
+
2373
+
2374
+ def item_review_topics(fields, asins, *, timeout=_TIMEOUT, sleep=time.sleep, now=time.monotonic):
2375
+ """Per child ASIN: its browse node and its topics by both sorts, interleaved on one token.
2376
+
2377
+ Returns `{"MENTIONS": [record], "STAR_RATING_IMPACT": [record], "browse_nodes": {asin: node}}`.
2378
+ """
2379
+ for operation in ("getItemBrowseNode", "getItemReviewTopics"):
2380
+ _feedback_admitted(operation)
2381
+ identifiers = _feedback_child_asins(asins)
2382
+ out = {**{sort: [] for sort in FEEDBACK_SORTS}, "browse_nodes": {}}
2383
+ if not identifiers:
2384
+ return out
2385
+ marketplace = fields.get("marketplace_id") or US_MARKETPLACE_ID
2386
+ node_path, topic_path = (OPERATIONS[name]["path"] for name in ("getItemBrowseNode", "getItemReviewTopics"))
2387
+
2388
+ def reads(asin):
2389
+ return [("browse_node", "getItemBrowseNode", node_path.replace("{asin}", asin), {"marketplaceId": marketplace}),
2390
+ *((sort, "getItemReviewTopics", topic_path.replace("{asin}", asin),
2391
+ {"marketplaceId": marketplace, "sortBy": sort}) for sort in FEEDBACK_SORTS)]
2392
+ for asin, label, body, reason in _feedback_reads(fields, identifiers, reads, subject="ASIN",
2393
+ timeout=timeout, sleep=sleep, now=now):
2394
+ if label == "browse_node":
2395
+ out["browse_nodes"][asin] = _browse_node_record(body, reason)
2396
+ else:
2397
+ out[label].append(_feedback_record("getItemReviewTopics", asin, marketplace, body, reason))
2398
+ return out
2399
+
2400
+
2401
+ def browse_node_review_topics(fields, node_ids, *, timeout=_TIMEOUT, sleep=time.sleep, now=time.monotonic):
2402
+ """Each browse node's top category topics by both sorts: `{sort: [record]}`; an absence carries `_no_feedback_reason`."""
2403
+ _feedback_admitted("getBrowseNodeReviewTopics")
2404
+ nodes = sorted(set(node_ids))
2405
+ # The id becomes a path segment, so only a numeric browse node id is admitted.
2406
+ if any(not isinstance(node, str) or not re.fullmatch(r"\d{1,20}", node) for node in nodes):
2407
+ raise AmazonError("Customer Feedback contains an invalid browse node ID. Retry after Amazon returns a numeric browse node.")
2408
+ out = {sort: [] for sort in FEEDBACK_SORTS}
2409
+ if not nodes:
2410
+ return out
2411
+ marketplace = fields.get("marketplace_id") or US_MARKETPLACE_ID
2412
+ path = OPERATIONS["getBrowseNodeReviewTopics"]["path"]
2413
+ reads = lambda node: [(sort, "getBrowseNodeReviewTopics", path.replace("{browseNodeId}", node),
2414
+ {"marketplaceId": marketplace, "sortBy": sort}) for sort in FEEDBACK_SORTS]
2415
+ for node, sort, body, reason in _feedback_reads(fields, nodes, reads, subject="browse node",
2416
+ timeout=timeout, sleep=sleep, now=now):
2417
+ if reason is not None:
2418
+ out[sort].append({"browseNodeId": node, "marketplaceId": marketplace, "_no_feedback_reason": reason})
2419
  continue
2420
+ if (not isinstance(body, dict) or str(body.get("browseNodeId")) != node
2421
+ or body.get("marketplaceId") != marketplace or not isinstance(body.get("topics"), dict)):
2422
+ raise AmazonError("Amazon getBrowseNodeReviewTopics returned an invalid response for the requested browse node and marketplace. Retry after Amazon restores its response.")
2423
  for key in ("positiveTopics", "negativeTopics"):
2424
+ topics = body["topics"].get(key)
2425
  if topics is not None and (not isinstance(topics, list) or any(not isinstance(t, dict) for t in topics)):
2426
+ raise AmazonError(f"Amazon getBrowseNodeReviewTopics returned invalid {key}. No partial snapshot was stored.")
2427
+ out[sort].append(body)
2428
+ return out
2429
 
2430
 
2431
  def group_a_records(fields, operation, *, start=None, end=None, asins=(),
 
2829
  if path.startswith("notifications/v1/"):
2830
  reason = (detail or "Amazon refused this notification operation.").replace(token, "[redacted]")
2831
  raise AmazonNotificationError(e.code, reason.replace("\u2014", ". ").replace("\u2013", " to ")) from None
2832
+ refusal = AmazonError(f"{_safe(url)} answered {e.code}" + (f": {detail}" if detail else ""))
2833
+ # Callers branch on the status attribute, never on the message text.
2834
+ refusal.status, refusal.vendor_message = e.code, detail
2835
+ raise refusal from None
2836
  except Exception as e: # noqa: BLE001
2837
  raise AmazonError(f"{_safe(url)} unreachable: {type(e).__name__}") from None
2838
 
platform/harness/amazon_schema.py CHANGED
@@ -563,10 +563,12 @@ TABLE_SPECS.update({
563
  for column, target, field, cardinality in _inbound_spec.links), grain="inbound_shipment_item"),
564
  "replenishment": _table_spec("replenishment", ("metric_id",),
565
  (("sku", "listings", "seller_sku", "many_to_one"),), grain="replenishment_metric"),
566
- "review_topics": _table_spec("review_topics", ("asin", "marketplace_id", "sentiment", "topic"),
567
- (("asin", "listings", "asin1", "many_to_one"),), grain="asin_sentiment_topic"),
568
  "review_trends": _table_spec("review_trends", ("asin", "marketplace_id", "sentiment", "topic", "month"),
569
  (("asin", "listings", "asin1", "many_to_one"),), grain="asin_sentiment_topic_month"),
 
 
570
  "sku_keywords": _table_spec("sku_keywords", ("keyword_id",),
571
  (("sku", "listing_items", "sku", "many_to_one"),), grain="sku_month_asin_query"),
572
  "keyword_ranks": _table_spec("keyword_ranks", ("rank_id",),
 
563
  for column, target, field, cardinality in _inbound_spec.links), grain="inbound_shipment_item"),
564
  "replenishment": _table_spec("replenishment", ("metric_id",),
565
  (("sku", "listings", "seller_sku", "many_to_one"),), grain="replenishment_metric"),
566
+ "review_topics": _table_spec("review_topics", ("asin", "marketplace_id", "sort_by", "sentiment", "topic"),
567
+ (("asin", "listings", "asin1", "many_to_one"),), grain="asin_sort_sentiment_topic"),
568
  "review_trends": _table_spec("review_trends", ("asin", "marketplace_id", "sentiment", "topic", "month"),
569
  (("asin", "listings", "asin1", "many_to_one"),), grain="asin_sentiment_topic_month"),
570
+ "browse_node_review_topics": _table_spec("browse_node_review_topics",
571
+ ("browse_node_id", "marketplace_id", "sort_by", "sentiment", "topic"), grain="browse_node_sort_sentiment_topic"),
572
  "sku_keywords": _table_spec("sku_keywords", ("keyword_id",),
573
  (("sku", "listing_items", "sku", "many_to_one"),), grain="sku_month_asin_query"),
574
  "keyword_ranks": _table_spec("keyword_ranks", ("rank_id",),
platform/harness/amazon_store.py CHANGED
@@ -85,6 +85,8 @@ SQP_PASS_REPORTS = 15
85
  SQP_RESUME_SECONDS = 900
86
  GROUP_A_TABLES = tuple(amazon_schema.GROUP_A_TABLES)
87
  TABLES += GROUP_A_TABLES
 
 
88
  DERIVED_TABLES = ("sku_keywords", "keyword_ranks")
89
  #: Snapshot reports whose changes are kept as versions in `amzn_<t>_history`.
90
  HISTORY_SOURCES = tuple(connector_spec.AMAZON_HISTORY_SOURCES)
@@ -5143,8 +5145,9 @@ def plain_refusal_reason(reason, *, status="", table="", operation=""):
5143
  return "The reconciliation check could not run. The last good rows are kept until it runs."
5144
  return (f"Reconciliation did not tie: {detail}. The last good rows are kept." if detail
5145
  else "Reconciliation did not tie. The last good rows are kept.")
5146
- feedback = table in ("review_topics", "review_trends")
5147
- subject = "review trends" if table == "review_trends" else "review topics"
 
5148
  if feedback and (str(status) in ("403", "NOT_OFFERED")
5149
  or re.search(r"answered\s+403\b|\(403\)", text)):
5150
  vendor = re.search(r"answered 403(?:: (.*?))?(?:; fix:|$)", text)
@@ -7034,7 +7037,7 @@ def _sync_problem(exc, table, operation, tenant, rt, report, log):
7034
  held = all(r.strip() in cx.ROLES_TAKEN
7035
  for r in str(role or "").replace(" / ", " + ").split(" + ") if r.strip())
7036
  held = held and not re.search(r"missing.*role|not granted|has not granted", str(exc), re.I)
7037
- fix = ("review Customer Feedback seller and marketplace eligibility and either Brand Analytics or Selling Partner Insights access, then reconnect Amazon" if table in ("review_topics", "review_trends") else
7038
  "review Amazon Fulfillment access for this seller account and reconnect Amazon" if table in ("outbound_fulfillment", "inbound_shipments") else
7039
  "review the Pricing role's access for this seller account and reconnect Amazon" if table == "fee_estimates_api" else
7040
  f"needs the {role} role; this application already holds it, so Amazon does not "
@@ -7380,6 +7383,8 @@ def _replace_period(con, table, rows, columns, column, start, end=None, marker=N
7380
  declared.update({name: kind for name, kind in REVIEW_TOPIC_TYPES.items() if name in columns})
7381
  if table == "review_trends":
7382
  declared.update({name: kind for name, kind in REVIEW_TREND_TYPES.items() if name in columns})
 
 
7383
  types = _ensure(con, mirror, columns, rows, declared=declared, preserve_rows=True)
7384
  predicate = f'"{column}" >= ? AND "{column}" < ?' if end else f'"{column}" = ?'
7385
  if column == "posted_date" and end:
@@ -8039,23 +8044,152 @@ REVIEW_TOPIC_METRICS = (
8039
  ("child_asin_metrics_most_mentions_number_of_mentions", ("childAsinMetrics", "mostMentions", "numberOfMentions")),
8040
  ("child_asin_metrics_least_mentions_number_of_mentions", ("childAsinMetrics", "leastMentions", "numberOfMentions")),
8041
  )
 
 
 
 
 
 
 
 
 
8042
  REVIEW_TOPIC_TYPES = {
8043
- **dict.fromkeys(("asin", "marketplace_id", "sentiment", "topic", "review_snippets", "subtopics",
8044
- "review_snippet_1", "review_snippet_2", "review_snippet_3"), "VARCHAR"),
 
8045
  **{name: "BIGINT" if name.endswith("_number_of_mentions") else "DOUBLE"
8046
  for name, _path in REVIEW_TOPIC_METRICS},
 
8047
  }
8048
 
8049
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
8050
  def _review_topic_evidence(records, asins, marketplace):
8051
  """Count raw vendor topics and retain their metrics before the row parser runs."""
8052
  import math
8053
- expected, counts, metrics, absent, seen = set(asins), {}, {}, [], set()
 
 
 
8054
  for record in records:
8055
- asin = record.get("asin")
8056
- if asin not in expected or asin in seen or record.get("marketplaceId") != marketplace:
8057
  raise ValueError("Amazon Customer Feedback returned a missing, duplicate or foreign ASIN or marketplace")
8058
- seen.add(asin)
8059
  no_content = record.get("_no_feedback_reason")
8060
  topics = record.get("topics")
8061
  if not no_content and not isinstance(topics, dict):
@@ -8066,13 +8200,15 @@ def _review_topic_evidence(records, asins, marketplace):
8066
  group = [] if group is None else group
8067
  if not isinstance(group, list):
8068
  raise ValueError("Amazon getItemReviewTopics returned an invalid sentiment topic list")
8069
- counts[json.dumps([asin, sentiment], separators=(",", ":"))] = len(group)
8070
  total += len(group)
8071
  for topic in group:
 
8072
  if (not isinstance(topic, dict) or not isinstance(topic.get("topic"), str) or not topic["topic"].strip()
8073
- or not isinstance(topic.get("asinMetrics"), dict)):
8074
- raise ValueError("Amazon review topic requires a topic name and asinMetrics object")
8075
- identity = json.dumps([asin, marketplace, sentiment, topic["topic"]], separators=(",", ":"))
 
8076
  if identity in metrics:
8077
  raise ValueError("Amazon returned duplicate topics for one ASIN and sentiment")
8078
  values = []
@@ -8092,58 +8228,63 @@ def _review_topic_evidence(records, asins, marketplace):
8092
  values.append(value)
8093
  metrics[identity] = values
8094
  if not total:
8095
- absent.append({"asin": asin, "code": "NO_CONTENT" if no_content else "NO_TOPICS",
 
8096
  "reason": no_content or "Amazon returned no positive or negative review topics for this ASIN and marketplace."})
8097
  if seen != expected:
8098
  raise ValueError("Amazon Customer Feedback did not return a response for every listing ASIN")
8099
- return {"count": sum(counts.values()), "source_counts": counts, "source_metrics": metrics,
8100
  "marketplace_id": marketplace, "absent_asins": absent}
8101
 
8102
 
8103
- def review_topic_rows(records, document):
8104
  """Flatten positive and negative topics while retaining null metrics and vendor percentage units."""
8105
  rows = []
8106
  for record in records:
 
8107
  for sentiment, key in (("positive", "positiveTopics"), ("negative", "negativeTopics")):
8108
  for topic in (record.get("topics") or {}).get(key) or []:
8109
  row = dict.fromkeys(REVIEW_TOPIC_TYPES)
8110
  row.update(_flatten_row(topic))
8111
- row.update(asin=record["asin"], marketplace_id=record["marketplaceId"], sentiment=sentiment)
 
 
8112
  row.update(_flatten_row({k: record[k] for k in ("itemName", "countryCode", "dateRange") if k in record}))
8113
- snippets = topic.get("reviewSnippets")
8114
- if snippets is not None and (not isinstance(snippets, list) or len(snippets) > 3
8115
- or any(not isinstance(snippet, str) for snippet in snippets)):
8116
- raise ValueError("Amazon review snippets must be null or a list of up to three strings")
8117
- for index, snippet in enumerate(snippets or [], 1):
8118
  row[f"review_snippet_{index}"] = snippet
 
8119
  rows.append(_origin(row, document, len(rows) + 1, "getItemReviewTopics"))
8120
  return rows
8121
 
8122
 
8123
  def _sync_review_topics(con, fields):
8124
- """Replace the complete listing-ASIN topic snapshot with same-pass vendor reconciliation."""
8125
- columns = _columns_of(con, "amzn_listings")
8126
- asins = ([row[0] for row in con.execute(
8127
- 'SELECT DISTINCT TRIM(asin1) FROM amzn_listings WHERE NULLIF(TRIM(asin1), \'\') IS NOT NULL ORDER BY 1').fetchall()]
8128
- if "asin1" in columns else [])
8129
  marketplace = fields.get("marketplace_id") or _client().US_MARKETPLACE_ID
8130
- records = _client().item_review_feedback(fields, asins)
 
 
 
8131
  evidence = _review_topic_evidence(records, asins, marketplace)
8132
  document = "operation:" + uuid.uuid4().hex
8133
- rows = review_topic_rows(records, document)
8134
  period = datetime.now(timezone.utc).date().isoformat()
8135
  for row in rows:
8136
  row["_snapshot_date"] = period
8137
  reason = ("Amazon Customer Feedback returned no review topics for this marketplace. "
8138
  "The per-ASIN reasons are recorded in the sync evidence. Amazon refreshes this data weekly."
8139
- if asins else "No source rows: the Amazon listings catalogue has no child ASINs. Sync Amazon listings first.")
8140
  marker = {"start": period, "end": period, "column": "_snapshot_date", **evidence,
 
8141
  "empty_reason": reason if not rows else ""}
8142
  count = _replace_period(con, "review_topics", rows,
8143
  sorted(set(REVIEW_TOPIC_TYPES) | {key for row in rows for key in row}),
8144
  "_snapshot_date", period, marker=marker, snapshot=True)
8145
  _record_state(con, entity_key("limit_review_topics"), "done", detail=json.dumps({
8146
- "reason": marker["empty_reason"], "absent_asins": evidence["absent_asins"]}))
8147
  return count
8148
 
8149
 
@@ -8151,13 +8292,52 @@ REVIEW_TREND_TYPES = {
8151
  **dict.fromkeys(("asin", "marketplace_id", "sentiment", "topic", "month",
8152
  "date_range_start_date", "date_range_end_date"), "VARCHAR"),
8153
  "asin_metrics_occurrence_percentage": "DOUBLE",
 
 
8154
  }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
8155
 
8156
 
8157
  def _review_trend_evidence(records, asins, marketplace):
8158
  """Count vendor trend points and retain source values before flattening or storing rows."""
8159
- import math
8160
  expected, seen, counts, metrics, absent = set(asins), set(), {}, {}, []
 
8161
  for record in records:
8162
  asin = record.get("asin")
8163
  if asin not in expected or asin in seen or record.get("marketplaceId") != marketplace:
@@ -8187,31 +8367,22 @@ def _review_trend_evidence(records, asins, marketplace):
8187
  counts[group] += len(topic["trendMetrics"])
8188
  for point in topic["trendMetrics"]:
8189
  if (not isinstance(point, dict) or not isinstance(point.get("dateRange"), dict)
8190
- or not isinstance(point.get("asinMetrics"), dict)):
8191
- raise ValueError("Amazon review trend requires dateRange and asinMetrics objects")
8192
- start, end = (point["dateRange"].get(name) for name in ("startDate", "endDate"))
8193
- if any(not isinstance(value, str) or not re.fullmatch(r"\d{4}-\d{2}-\d{2}", value)
8194
- for value in (start, end)):
8195
- raise ValueError("Amazon review trend requires ISO calendar dates")
8196
- first, last = datetime.strptime(start, "%Y-%m-%d"), datetime.strptime(end, "%Y-%m-%d")
8197
- if first > last or start[:7] != end[:7]:
8198
- raise ValueError("Amazon review trend dateRange must identify one calendar month")
8199
- identity = json.dumps([asin, marketplace, sentiment, topic["topic"], start[:7]],
8200
  separators=(",", ":"))
8201
  if identity in metrics:
8202
  raise ValueError("Amazon returned duplicate trend points for one ASIN, topic and month")
8203
- value = point["asinMetrics"].get("occurrencePercentage")
8204
- if value is not None and (isinstance(value, bool) or not isinstance(value, (int, float))
8205
- or not math.isfinite(value) or not 0 <= value <= 100):
8206
- raise ValueError("Amazon review trend returned an invalid occurrence percentage")
8207
- metrics[identity] = [start, end, value]
8208
  total += counts[group]
8209
  if not total:
8210
  absent.append({"asin": asin, "code": "NO_CONTENT" if no_content else "NO_TRENDS",
8211
  "reason": no_content or "Amazon returned no positive or negative review trend points for this ASIN and marketplace."})
8212
  if seen != expected:
8213
  raise ValueError("Amazon Customer Feedback did not return a response for every listing ASIN")
8214
- return {"count": sum(counts.values()), "source_counts": counts, "source_metrics": metrics,
8215
  "marketplace_id": marketplace, "absent_asins": absent}
8216
 
8217
 
@@ -8222,23 +8393,22 @@ def review_trend_rows(records, document):
8222
  for sentiment, key in (("positive", "positiveTopics"), ("negative", "negativeTopics")):
8223
  for topic in (record.get("reviewTrends") or {}).get(key) or []:
8224
  for point in topic["trendMetrics"]:
8225
- dates = point["dateRange"]
8226
  row = {"asin": record["asin"], "marketplace_id": record["marketplaceId"],
8227
- "sentiment": sentiment, "topic": topic["topic"], "month": dates["startDate"][:7],
8228
- "date_range_start_date": dates["startDate"], "date_range_end_date": dates["endDate"],
8229
- "asin_metrics_occurrence_percentage": point["asinMetrics"].get("occurrencePercentage")}
8230
  rows.append(_origin(row, document, len(rows) + 1, "getItemReviewTrends"))
8231
  return rows
8232
 
8233
 
8234
  def _sync_review_trends(con, fields):
8235
  """Replace the full six-month review snapshot with independently counted source evidence."""
8236
- columns = _columns_of(con, "amzn_listings")
8237
- asins = ([row[0] for row in con.execute(
8238
- 'SELECT DISTINCT TRIM(asin1) FROM amzn_listings WHERE NULLIF(TRIM(asin1), \'\') IS NOT NULL ORDER BY 1').fetchall()]
8239
- if "asin1" in columns else [])
8240
  marketplace = fields.get("marketplace_id") or _client().US_MARKETPLACE_ID
8241
  records = _client().item_review_feedback(fields, asins, operation="getItemReviewTrends")
 
 
8242
  evidence = _review_trend_evidence(records, asins, marketplace)
8243
  rows = review_trend_rows(records, "operation:" + uuid.uuid4().hex)
8244
  period = datetime.now(timezone.utc).date().isoformat()
@@ -8246,13 +8416,149 @@ def _sync_review_trends(con, fields):
8246
  row["_snapshot_date"] = period
8247
  reason = ("Amazon Customer Feedback returned no review trends for this marketplace. "
8248
  "The per-ASIN reasons are recorded in the sync evidence. Amazon refreshes this data weekly."
8249
- if asins else "No source rows: the Amazon listings catalogue has no child ASINs. Sync Amazon listings first.")
8250
  marker = {"start": period, "end": period, "column": "_snapshot_date", **evidence,
 
8251
  "empty_reason": reason if not rows else ""}
8252
  count = _replace_period(con, "review_trends", rows, REVIEW_TREND_TYPES,
8253
  "_snapshot_date", period, marker=marker, snapshot=True)
8254
  _record_state(con, entity_key("limit_review_trends"), "done", detail=json.dumps({
8255
- "reason": marker["empty_reason"], "absent_asins": evidence["absent_asins"]}))
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
8256
  return count
8257
 
8258
 
@@ -8264,6 +8570,8 @@ def _sync_margin_window(con, table, fields, w0, w1, *, max_wait, log, operations
8264
  return _sync_review_topics(con, fields)
8265
  if table == "review_trends":
8266
  return _sync_review_trends(con, fields)
 
 
8267
  if table == "economics":
8268
  start, end = _utc(w0).date().isoformat(), _utc(w1).date().isoformat()
8269
  if start >= end:
@@ -8673,7 +8981,7 @@ def _validate_margin(con, table):
8673
  identity = json.dumps([row.get("asin"), row.get("marketplace_id"), row.get("sentiment"),
8674
  row.get("topic"), row.get("month")], separators=(",", ":"))
8675
  metrics[identity] = [row.get("date_range_start_date"), row.get("date_range_end_date"),
8676
- row.get("asin_metrics_occurrence_percentage")]
8677
  total = con.execute(f"SELECT COUNT(*) FROM {table_name(table)}").fetchone()[0]
8678
  trends_match = (marker["count"] == len(rows) == len(metrics) == total
8679
  and counts == marker["source_counts"] and metrics == marker["source_metrics"])
@@ -8684,10 +8992,10 @@ def _validate_margin(con, table):
8684
  counts = dict.fromkeys(marker["source_counts"], 0)
8685
  metrics = {}
8686
  for row in rows:
8687
- group = json.dumps([row.get("asin"), row.get("sentiment")], separators=(",", ":"))
8688
  counts[group] = counts.get(group, 0) + 1
8689
- identity = json.dumps([row.get("asin"), row.get("marketplace_id"), row.get("sentiment"), row.get("topic")],
8690
- separators=(",", ":"))
8691
  metrics[identity] = [row.get(name) for name, _path in REVIEW_TOPIC_METRICS]
8692
  total = con.execute(f"SELECT COUNT(*) FROM {table_name(table)}").fetchone()[0]
8693
  matches = (marker["count"] == len(rows) == len(metrics) == total
@@ -8695,6 +9003,22 @@ def _validate_margin(con, table):
8695
  return {"state": "ok" if matches else "refused", "expected": marker["count"], "actual": len(rows),
8696
  "reason": (marker["empty_reason"] if matches and not rows else
8697
  f"Amazon returned {marker['count']} review topics. Stored {len(rows)} topics with per-ASIN, sentiment and metric reconciliation.")}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
8698
  if table == "fee_estimates_api":
8699
  expected, actual = marker["count"], len(rows)
8700
  identities = {row.get("estimate_id") for row in rows}
@@ -9428,8 +9752,9 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
9428
  # A watermark is written when a table's history is complete, so its next window is
9429
  # bounded; deferring it too froze every loaded table behind one refused report.
9430
  for table in want:
 
9431
  if ((read_watermark(con, table) and (not refresh_loaded or table not in repairs))
9432
- or table in completed
9433
  or (table == "settlements" and not refresh_loaded)):
9434
  continue
9435
  report["tables"][table] = {"state": "deferred", "reason":
@@ -9442,7 +9767,7 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
9442
  _record_state(con, entity_key("sync_tick_" + table), "deferred", tick.get("rows") or 0, tick.get("detail"))
9443
  want = [table for table in want if (read_watermark(con, table)
9444
  and (not refresh_loaded or table not in repairs))
9445
- or table in completed
9446
  or (table == "settlements" and not refresh_loaded)]
9447
  if scheduled and not first_load:
9448
  ticks = {table: _read_state(con, entity_key("sync_tick_" + table)) for table in TABLES}
@@ -9470,6 +9795,10 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
9470
  elif not first_load:
9471
  # The vendor's report floor also applies to an operator's Sync now click.
9472
  for table in list(want):
 
 
 
 
9473
  reason = _interval_reason(table, _read_state(con, entity_key("sync_tick_" + table)),
9474
  datetime.now(timezone.utc))
9475
  if reason:
@@ -10580,7 +10909,7 @@ def _validate_account(tenant_key="royal-imports", rt=None, log=print):
10580
  out[table] = {"state": "unproven", "reason": "table has never synced",
10581
  "reconciles": spec.get("validates_against")}
10582
  continue
10583
- if table in ("fee_estimates_api", "replenishment", "inbound_shipments", "review_topics", "review_trends"):
10584
  out[table] = {"rows": n, "reconciles": spec.get("validates_against"),
10585
  **_validate_margin(con, table)}
10586
  continue
@@ -11266,6 +11595,24 @@ def snapshot_sync_selftest(log=print, check=None):
11266
  "numberOfMentions": generation[0] + 2, "occurrencePercentage": 12.5,
11267
  "starRatingImpact": 0.25}}], "negativeTopics": None}} for asin in asins]
11268
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
11269
  def snapshot_counts(requests):
11270
  return Counter(table for table, _kwargs in requests if table in snapshots)
11271
 
@@ -11305,6 +11652,9 @@ def snapshot_sync_selftest(log=print, check=None):
11305
  patch.object(cx, "outbound_fulfillment_orders", side_effect=outbound_records),
11306
  patch.object(cx, "inbound_shipment_items", side_effect=inbound_records),
11307
  patch.object(cx, "item_review_feedback", side_effect=feedback_records),
 
 
 
11308
  patch.object(cx, "replenishment_metrics", side_effect=replenishment_records),
11309
  patch.object(cx, "awd_inventory", return_value=[]),
11310
  patch.object(cx, "access_token", side_effect=AssertionError("offline snapshot sync fixture")),
@@ -12797,6 +13147,16 @@ def group_a_selftest(log=print, check=None):
12797
  "occurrencePercentage": 12.5, "starRatingImpact": 0.2}}],
12798
  "negativeTopics": [{"topic": "Fixture assembly", "asinMetrics": {"numberOfMentions": 2,
12799
  "occurrencePercentage": 6.25, "starRatingImpact": -0.1}}]}}]
 
 
 
 
 
 
 
 
 
 
12800
 
12801
  try:
12802
  with patch.object(cx, "access_token", side_effect=AssertionError("No LWA in fixtures")), \
@@ -12807,6 +13167,9 @@ def group_a_selftest(log=print, check=None):
12807
  patch.object(cx, "inbound_shipment_items", return_value=inbound), \
12808
  patch.object(cx, "replenishment_metrics", return_value=replenishment), \
12809
  patch.object(cx, "item_review_feedback", return_value=feedback), \
 
 
 
12810
  patch.object(module, "_fee_estimate_inputs", return_value=[fee_input]), \
12811
  patch.object(cx, "fee_estimates", side_effect=lambda _f, estimates: _fixture_fee_records(estimates)):
12812
  for table in GROUP_A_TABLES:
@@ -13037,6 +13400,72 @@ def group_a_selftest(log=print, check=None):
13037
  finally:
13038
  con.close()
13039
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
13040
  # Client pagination: an empty middle page must not end the traversal.
13041
  for operation, interval in cx.ENTITY_INTERVALS.items():
13042
  if operation == "getMyFeesEstimates":
 
85
  SQP_RESUME_SECONDS = 900
86
  GROUP_A_TABLES = tuple(amazon_schema.GROUP_A_TABLES)
87
  TABLES += GROUP_A_TABLES
88
+ #: Customer Feedback entity snapshots: weekly current state, no history window.
89
+ FEEDBACK_TABLES = ("review_topics", "review_trends", "browse_node_review_topics")
90
  DERIVED_TABLES = ("sku_keywords", "keyword_ranks")
91
  #: Snapshot reports whose changes are kept as versions in `amzn_<t>_history`.
92
  HISTORY_SOURCES = tuple(connector_spec.AMAZON_HISTORY_SOURCES)
 
5145
  return "The reconciliation check could not run. The last good rows are kept until it runs."
5146
  return (f"Reconciliation did not tie: {detail}. The last good rows are kept." if detail
5147
  else "Reconciliation did not tie. The last good rows are kept.")
5148
+ feedback = table in FEEDBACK_TABLES
5149
+ subject = {"review_trends": "review trends",
5150
+ "browse_node_review_topics": "category review topics"}.get(table, "review topics")
5151
  if feedback and (str(status) in ("403", "NOT_OFFERED")
5152
  or re.search(r"answered\s+403\b|\(403\)", text)):
5153
  vendor = re.search(r"answered 403(?:: (.*?))?(?:; fix:|$)", text)
 
7037
  held = all(r.strip() in cx.ROLES_TAKEN
7038
  for r in str(role or "").replace(" / ", " + ").split(" + ") if r.strip())
7039
  held = held and not re.search(r"missing.*role|not granted|has not granted", str(exc), re.I)
7040
+ fix = ("review Customer Feedback seller and marketplace eligibility and either Brand Analytics or Selling Partner Insights access, then reconnect Amazon" if table in FEEDBACK_TABLES else
7041
  "review Amazon Fulfillment access for this seller account and reconnect Amazon" if table in ("outbound_fulfillment", "inbound_shipments") else
7042
  "review the Pricing role's access for this seller account and reconnect Amazon" if table == "fee_estimates_api" else
7043
  f"needs the {role} role; this application already holds it, so Amazon does not "
 
7383
  declared.update({name: kind for name, kind in REVIEW_TOPIC_TYPES.items() if name in columns})
7384
  if table == "review_trends":
7385
  declared.update({name: kind for name, kind in REVIEW_TREND_TYPES.items() if name in columns})
7386
+ if table == "browse_node_review_topics":
7387
+ declared.update({name: kind for name, kind in BROWSE_NODE_TOPIC_TYPES.items() if name in columns})
7388
  types = _ensure(con, mirror, columns, rows, declared=declared, preserve_rows=True)
7389
  predicate = f'"{column}" >= ? AND "{column}" < ?' if end else f'"{column}" = ?'
7390
  if column == "posted_date" and end:
 
8044
  ("child_asin_metrics_most_mentions_number_of_mentions", ("childAsinMetrics", "mostMentions", "numberOfMentions")),
8045
  ("child_asin_metrics_least_mentions_number_of_mentions", ("childAsinMetrics", "leastMentions", "numberOfMentions")),
8046
  )
8047
+ #: Amazon returns at most five subtopics per topic and three snippets per topic or subtopic.
8048
+ REVIEW_SUBTOPICS, REVIEW_SNIPPETS = 5, 3
8049
+ _SUBTOPIC_TYPES = {
8050
+ **{f"subtopic_{i}": "VARCHAR" for i in range(1, REVIEW_SUBTOPICS + 1)},
8051
+ **{f"subtopic_{i}_number_of_mentions": "BIGINT" for i in range(1, REVIEW_SUBTOPICS + 1)},
8052
+ **{f"subtopic_{i}_occurrence_percentage": "DOUBLE" for i in range(1, REVIEW_SUBTOPICS + 1)},
8053
+ **{f"subtopic_{i}_review_snippet_{j}": "VARCHAR"
8054
+ for i in range(1, REVIEW_SUBTOPICS + 1) for j in range(1, REVIEW_SNIPPETS + 1)},
8055
+ }
8056
  REVIEW_TOPIC_TYPES = {
8057
+ **dict.fromkeys(("asin", "marketplace_id", "sort_by", "sentiment", "topic", "review_snippets", "subtopics",
8058
+ "review_snippet_1", "review_snippet_2", "review_snippet_3",
8059
+ "browse_node_id", "browse_node_name"), "VARCHAR"),
8060
  **{name: "BIGINT" if name.endswith("_number_of_mentions") else "DOUBLE"
8061
  for name, _path in REVIEW_TOPIC_METRICS},
8062
+ **_SUBTOPIC_TYPES,
8063
  }
8064
 
8065
 
8066
+ def _feedback_asins(con):
8067
+ """The tenant's active child ASINs: active listings minus every parent in the latest catalogue snapshot."""
8068
+ columns = _columns_of(con, "amzn_listings")
8069
+ if "asin1" not in columns:
8070
+ return []
8071
+ active = " AND LOWER(TRIM(CAST(status AS VARCHAR))) = 'active'" if "status" in columns else ""
8072
+ asins = [row[0] for row in con.execute(
8073
+ "SELECT DISTINCT TRIM(asin1) FROM amzn_listings WHERE NULLIF(TRIM(asin1), '') IS NOT NULL"
8074
+ + active + " ORDER BY 1").fetchall()]
8075
+ catalog = _columns_of(con, "amzn_catalog_items")
8076
+ if "parent_asin" not in catalog:
8077
+ return asins
8078
+ other = " AND TRIM(CAST(parent_asin AS VARCHAR)) <> TRIM(CAST(asin AS VARCHAR))" if "asin" in catalog else ""
8079
+ parents = {row[0] for row in con.execute(
8080
+ "SELECT DISTINCT TRIM(CAST(parent_asin AS VARCHAR)) FROM ("
8081
+ + _latest_account_snapshot_sql(catalog, "amzn_catalog_items")
8082
+ + ") AS current_catalog WHERE NULLIF(TRIM(CAST(parent_asin AS VARCHAR)), '') IS NOT NULL" + other).fetchall()}
8083
+ return [asin for asin in asins if asin not in parents]
8084
+
8085
+
8086
+ FEEDBACK_NO_ASINS = ("No source rows: the Amazon listings catalogue has no active child ASINs. "
8087
+ "Sync Amazon listings first.")
8088
+
8089
+
8090
+ def _feedback_number(value, *, count=False, percentage=False):
8091
+ """A Customer Feedback metric: null, or a finite number; a count is a nonnegative integer."""
8092
+ import math
8093
+ if value is None:
8094
+ return None
8095
+ if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value):
8096
+ raise ValueError("Amazon review topic returned a nonnumeric metric")
8097
+ if count and (type(value) is not int or value < 0):
8098
+ raise ValueError("Amazon review topic returned an invalid mention count")
8099
+ if percentage and not 0 <= value <= 100:
8100
+ raise ValueError("Amazon review topic returned an invalid occurrence percentage")
8101
+ return value
8102
+
8103
+
8104
+ def _feedback_snippets(snippets):
8105
+ if snippets is not None and (not isinstance(snippets, list) or len(snippets) > REVIEW_SNIPPETS
8106
+ or any(not isinstance(snippet, str) for snippet in snippets)):
8107
+ raise ValueError("Amazon review snippets must be null or a list of up to three strings")
8108
+ return snippets or []
8109
+
8110
+
8111
+ def _subtopic_columns(topic, *, mentions=True):
8112
+ """Up to five subtopics as columns: name, metrics and up to three snippets each."""
8113
+ subtopics = topic.get("subtopics")
8114
+ if subtopics is not None and (not isinstance(subtopics, list) or len(subtopics) > REVIEW_SUBTOPICS
8115
+ or any(not isinstance(sub, dict) for sub in subtopics)):
8116
+ raise ValueError("Amazon review subtopics must be null or a list of up to five objects")
8117
+ out = {}
8118
+ for i, sub in enumerate(subtopics or [], 1):
8119
+ metrics = sub.get("metrics")
8120
+ if (not isinstance(sub.get("subtopic"), str) or not sub["subtopic"].strip()
8121
+ or (metrics is not None and not isinstance(metrics, dict))):
8122
+ raise ValueError("Amazon review subtopic requires a subtopic name and a metrics object")
8123
+ metrics = metrics or {}
8124
+ out[f"subtopic_{i}"] = sub["subtopic"]
8125
+ if mentions:
8126
+ out[f"subtopic_{i}_number_of_mentions"] = _feedback_number(metrics.get("numberOfMentions"), count=True)
8127
+ out[f"subtopic_{i}_occurrence_percentage"] = _feedback_number(metrics.get("occurrencePercentage"))
8128
+ for j, snippet in enumerate(_feedback_snippets(sub.get("reviewSnippets")), 1):
8129
+ out[f"subtopic_{i}_review_snippet_{j}"] = snippet
8130
+ return out
8131
+
8132
+
8133
+ def _null_unresolved_metrics(records):
8134
+ """Read an unfilled Amazon placeholder (a "$" string inside a metrics object) as null; return how many."""
8135
+ count = 0
8136
+
8137
+ def walk(node, inside):
8138
+ nonlocal count
8139
+ items = node.items() if isinstance(node, dict) else enumerate(node) if isinstance(node, list) else ()
8140
+ for key, value in list(items):
8141
+ if inside and isinstance(value, str) and value.startswith("$"):
8142
+ node[key] = None
8143
+ count += 1
8144
+ elif isinstance(value, (dict, list)):
8145
+ walk(value, inside or (isinstance(key, str) and key.endswith(("Metrics", "metrics"))))
8146
+ walk(records, False)
8147
+ return count
8148
+
8149
+
8150
+ def _drop_repeated_topics(records):
8151
+ """Drop a topic Amazon repeats with identical metrics in one sorted answer; return how many. A conflicting repeat stays."""
8152
+ dropped = 0
8153
+ for record in records:
8154
+ topics = record.get("topics")
8155
+ if not isinstance(topics, dict):
8156
+ continue
8157
+ for key in ("positiveTopics", "negativeTopics"):
8158
+ group = topics.get(key)
8159
+ if not isinstance(group, list):
8160
+ continue
8161
+ kept, first = [], {}
8162
+ for topic in group:
8163
+ name = topic.get("topic") if isinstance(topic, dict) else None
8164
+ metrics = (json.dumps({k: v for k, v in topic.items() if k.endswith("Metrics")}, sort_keys=True, default=str)
8165
+ if isinstance(topic, dict) else None)
8166
+ if isinstance(name, str) and name in first and first[name] == metrics:
8167
+ dropped += 1
8168
+ continue
8169
+ if isinstance(name, str):
8170
+ first.setdefault(name, metrics)
8171
+ kept.append(topic)
8172
+ topics[key] = kept
8173
+ return dropped
8174
+
8175
+
8176
+ def _sorted_feedback(read):
8177
+ """Stamp each record with the sort it answered, in FEEDBACK_SORTS order."""
8178
+ return [{**record, "_sort_by": sort} for sort in _client().FEEDBACK_SORTS for record in read.get(sort) or []]
8179
+
8180
+
8181
  def _review_topic_evidence(records, asins, marketplace):
8182
  """Count raw vendor topics and retain their metrics before the row parser runs."""
8183
  import math
8184
+ sorts = _client().FEEDBACK_SORTS
8185
+ expected = {(asin, sort) for asin in asins for sort in sorts}
8186
+ counts, metrics, absent, seen = {}, {}, [], set()
8187
+ metricless = 0
8188
  for record in records:
8189
+ asin, sort = record.get("asin"), record.get("_sort_by")
8190
+ if (asin, sort) not in expected or (asin, sort) in seen or record.get("marketplaceId") != marketplace:
8191
  raise ValueError("Amazon Customer Feedback returned a missing, duplicate or foreign ASIN or marketplace")
8192
+ seen.add((asin, sort))
8193
  no_content = record.get("_no_feedback_reason")
8194
  topics = record.get("topics")
8195
  if not no_content and not isinstance(topics, dict):
 
8200
  group = [] if group is None else group
8201
  if not isinstance(group, list):
8202
  raise ValueError("Amazon getItemReviewTopics returned an invalid sentiment topic list")
8203
+ counts[json.dumps([asin, sort, sentiment], separators=(",", ":"))] = len(group)
8204
  total += len(group)
8205
  for topic in group:
8206
+ # Amazon omits asinMetrics on some real topics: the topic stays, its metrics read null and are counted.
8207
  if (not isinstance(topic, dict) or not isinstance(topic.get("topic"), str) or not topic["topic"].strip()
8208
+ or not isinstance(topic.get("asinMetrics", {}), (dict, type(None)))):
8209
+ raise ValueError("Amazon review topic requires a topic name and an asinMetrics object or none")
8210
+ metricless += not isinstance(topic.get("asinMetrics"), dict)
8211
+ identity = json.dumps([asin, marketplace, sort, sentiment, topic["topic"]], separators=(",", ":"))
8212
  if identity in metrics:
8213
  raise ValueError("Amazon returned duplicate topics for one ASIN and sentiment")
8214
  values = []
 
8228
  values.append(value)
8229
  metrics[identity] = values
8230
  if not total:
8231
+ absent.append({"asin": asin, "sort_by": sort,
8232
+ "code": "NO_CONTENT" if no_content else "NO_TOPICS",
8233
  "reason": no_content or "Amazon returned no positive or negative review topics for this ASIN and marketplace."})
8234
  if seen != expected:
8235
  raise ValueError("Amazon Customer Feedback did not return a response for every listing ASIN")
8236
+ return {"count": sum(counts.values()), "metricless": metricless, "source_counts": counts, "source_metrics": metrics,
8237
  "marketplace_id": marketplace, "absent_asins": absent}
8238
 
8239
 
8240
+ def review_topic_rows(records, document, browse_nodes=None):
8241
  """Flatten positive and negative topics while retaining null metrics and vendor percentage units."""
8242
  rows = []
8243
  for record in records:
8244
+ node = (browse_nodes or {}).get(record["asin"]) or {}
8245
  for sentiment, key in (("positive", "positiveTopics"), ("negative", "negativeTopics")):
8246
  for topic in (record.get("topics") or {}).get(key) or []:
8247
  row = dict.fromkeys(REVIEW_TOPIC_TYPES)
8248
  row.update(_flatten_row(topic))
8249
+ row.update(asin=record["asin"], marketplace_id=record["marketplaceId"], sentiment=sentiment,
8250
+ sort_by=record.get("_sort_by"), browse_node_id=node.get("browse_node_id"),
8251
+ browse_node_name=node.get("browse_node_name"))
8252
  row.update(_flatten_row({k: record[k] for k in ("itemName", "countryCode", "dateRange") if k in record}))
8253
+ # All three Customer Feedback tables store the review period as a calendar date.
8254
+ for name in ("date_range_start_date", "date_range_end_date"):
8255
+ row[name] = str(row[name])[:10] if row.get(name) is not None else None
8256
+ for index, snippet in enumerate(_feedback_snippets(topic.get("reviewSnippets")), 1):
 
8257
  row[f"review_snippet_{index}"] = snippet
8258
+ row.update(_subtopic_columns(topic))
8259
  rows.append(_origin(row, document, len(rows) + 1, "getItemReviewTopics"))
8260
  return rows
8261
 
8262
 
8263
  def _sync_review_topics(con, fields):
8264
+ """Replace the complete active child ASIN topic snapshot, both sorts, with same-pass vendor reconciliation."""
8265
+ asins = _feedback_asins(con)
 
 
 
8266
  marketplace = fields.get("marketplace_id") or _client().US_MARKETPLACE_ID
8267
+ read = _client().item_review_topics(fields, asins)
8268
+ records = _sorted_feedback(read)
8269
+ unresolved = _null_unresolved_metrics(records)
8270
+ repeated = _drop_repeated_topics(records)
8271
  evidence = _review_topic_evidence(records, asins, marketplace)
8272
  document = "operation:" + uuid.uuid4().hex
8273
+ rows = review_topic_rows(records, document, read.get("browse_nodes"))
8274
  period = datetime.now(timezone.utc).date().isoformat()
8275
  for row in rows:
8276
  row["_snapshot_date"] = period
8277
  reason = ("Amazon Customer Feedback returned no review topics for this marketplace. "
8278
  "The per-ASIN reasons are recorded in the sync evidence. Amazon refreshes this data weekly."
8279
+ if asins else FEEDBACK_NO_ASINS)
8280
  marker = {"start": period, "end": period, "column": "_snapshot_date", **evidence,
8281
+ "unresolved_metrics": unresolved, "repeated_topics": repeated,
8282
  "empty_reason": reason if not rows else ""}
8283
  count = _replace_period(con, "review_topics", rows,
8284
  sorted(set(REVIEW_TOPIC_TYPES) | {key for row in rows for key in row}),
8285
  "_snapshot_date", period, marker=marker, snapshot=True)
8286
  _record_state(con, entity_key("limit_review_topics"), "done", detail=json.dumps({
8287
+ "reason": marker["empty_reason"], "absent_asins": evidence["absent_asins"], "unresolved_metrics": unresolved, "repeated_topics": repeated}))
8288
  return count
8289
 
8290
 
 
8292
  **dict.fromkeys(("asin", "marketplace_id", "sentiment", "topic", "month",
8293
  "date_range_start_date", "date_range_end_date"), "VARCHAR"),
8294
  "asin_metrics_occurrence_percentage": "DOUBLE",
8295
+ "parent_asin_metrics_occurrence_percentage": "DOUBLE",
8296
+ "browse_node_metrics_occurrence_percentage_all_products": "DOUBLE",
8297
  }
8298
+ _TREND_METRICS = (("asin_metrics_occurrence_percentage", ("asinMetrics", "occurrencePercentage")),
8299
+ ("parent_asin_metrics_occurrence_percentage", ("parentAsinMetrics", "occurrencePercentage")),
8300
+ ("browse_node_metrics_occurrence_percentage_all_products",
8301
+ ("browseNodeMetrics", "occurrencePercentage", "allProducts")))
8302
+
8303
+
8304
+ def _trend_window(date_range):
8305
+ """`(start, end, month)` of one trend point: ISO dates, start before end, month the last one inside the window."""
8306
+ start, end = (date_range.get(name) for name in ("startDate", "endDate"))
8307
+ days = []
8308
+ for value in (start, end):
8309
+ if not isinstance(value, str) or not re.fullmatch(r"\d{4}-\d{2}-\d{2}(T[0-9:.]+(Z|[+-]\d{2}:\d{2})?)?", value):
8310
+ raise ValueError("Amazon review trend requires ISO calendar dates")
8311
+ try:
8312
+ days.append(datetime.strptime(value[:10], "%Y-%m-%d"))
8313
+ if len(value) > 10:
8314
+ datetime.fromisoformat(value.replace("Z", "+00:00"))
8315
+ except ValueError:
8316
+ raise ValueError("Amazon review trend requires ISO calendar dates") from None
8317
+ if days[0] >= days[1]:
8318
+ raise ValueError("Amazon review trend dateRange must start before it ends")
8319
+ return start[:10], end[:10], (days[1] - timedelta(days=1)).strftime("%Y-%m")
8320
+
8321
+
8322
+ def _trend_values(point):
8323
+ """The ASIN, parent and browse node occurrence percentages of one trend point."""
8324
+ import math
8325
+ values = []
8326
+ for _name, path in _TREND_METRICS:
8327
+ value = point
8328
+ for segment in path:
8329
+ value = value.get(segment) if isinstance(value, dict) else None
8330
+ if value is not None and (isinstance(value, bool) or not isinstance(value, (int, float))
8331
+ or not math.isfinite(value) or not 0 <= value <= 100):
8332
+ raise ValueError("Amazon review trend returned an invalid occurrence percentage")
8333
+ values.append(value)
8334
+ return values
8335
 
8336
 
8337
  def _review_trend_evidence(records, asins, marketplace):
8338
  """Count vendor trend points and retain source values before flattening or storing rows."""
 
8339
  expected, seen, counts, metrics, absent = set(asins), set(), {}, {}, []
8340
+ metricless = 0
8341
  for record in records:
8342
  asin = record.get("asin")
8343
  if asin not in expected or asin in seen or record.get("marketplaceId") != marketplace:
 
8367
  counts[group] += len(topic["trendMetrics"])
8368
  for point in topic["trendMetrics"]:
8369
  if (not isinstance(point, dict) or not isinstance(point.get("dateRange"), dict)
8370
+ or not isinstance(point.get("asinMetrics", {}), (dict, type(None)))):
8371
+ raise ValueError("Amazon review trend requires a dateRange object and an asinMetrics object or none")
8372
+ metricless += not isinstance(point.get("asinMetrics"), dict)
8373
+ start, end, month = _trend_window(point["dateRange"])
8374
+ identity = json.dumps([asin, marketplace, sentiment, topic["topic"], month],
 
 
 
 
 
8375
  separators=(",", ":"))
8376
  if identity in metrics:
8377
  raise ValueError("Amazon returned duplicate trend points for one ASIN, topic and month")
8378
+ metrics[identity] = [start, end, *_trend_values(point)]
 
 
 
 
8379
  total += counts[group]
8380
  if not total:
8381
  absent.append({"asin": asin, "code": "NO_CONTENT" if no_content else "NO_TRENDS",
8382
  "reason": no_content or "Amazon returned no positive or negative review trend points for this ASIN and marketplace."})
8383
  if seen != expected:
8384
  raise ValueError("Amazon Customer Feedback did not return a response for every listing ASIN")
8385
+ return {"count": sum(counts.values()), "metricless": metricless, "source_counts": counts, "source_metrics": metrics,
8386
  "marketplace_id": marketplace, "absent_asins": absent}
8387
 
8388
 
 
8393
  for sentiment, key in (("positive", "positiveTopics"), ("negative", "negativeTopics")):
8394
  for topic in (record.get("reviewTrends") or {}).get(key) or []:
8395
  for point in topic["trendMetrics"]:
8396
+ start, end, month = _trend_window(point["dateRange"])
8397
  row = {"asin": record["asin"], "marketplace_id": record["marketplaceId"],
8398
+ "sentiment": sentiment, "topic": topic["topic"], "month": month,
8399
+ "date_range_start_date": start, "date_range_end_date": end,
8400
+ **dict(zip((name for name, _path in _TREND_METRICS), _trend_values(point)))}
8401
  rows.append(_origin(row, document, len(rows) + 1, "getItemReviewTrends"))
8402
  return rows
8403
 
8404
 
8405
  def _sync_review_trends(con, fields):
8406
  """Replace the full six-month review snapshot with independently counted source evidence."""
8407
+ asins = _feedback_asins(con)
 
 
 
8408
  marketplace = fields.get("marketplace_id") or _client().US_MARKETPLACE_ID
8409
  records = _client().item_review_feedback(fields, asins, operation="getItemReviewTrends")
8410
+ unresolved = _null_unresolved_metrics(records)
8411
+ repeated = _drop_repeated_topics(records)
8412
  evidence = _review_trend_evidence(records, asins, marketplace)
8413
  rows = review_trend_rows(records, "operation:" + uuid.uuid4().hex)
8414
  period = datetime.now(timezone.utc).date().isoformat()
 
8416
  row["_snapshot_date"] = period
8417
  reason = ("Amazon Customer Feedback returned no review trends for this marketplace. "
8418
  "The per-ASIN reasons are recorded in the sync evidence. Amazon refreshes this data weekly."
8419
+ if asins else FEEDBACK_NO_ASINS)
8420
  marker = {"start": period, "end": period, "column": "_snapshot_date", **evidence,
8421
+ "unresolved_metrics": unresolved, "repeated_topics": repeated,
8422
  "empty_reason": reason if not rows else ""}
8423
  count = _replace_period(con, "review_trends", rows, REVIEW_TREND_TYPES,
8424
  "_snapshot_date", period, marker=marker, snapshot=True)
8425
  _record_state(con, entity_key("limit_review_trends"), "done", detail=json.dumps({
8426
+ "reason": marker["empty_reason"], "absent_asins": evidence["absent_asins"], "unresolved_metrics": unresolved, "repeated_topics": repeated}))
8427
+ return count
8428
+
8429
+
8430
+ BROWSE_NODE_TOPIC_METRICS = (
8431
+ ("browse_node_metrics_occurrence_percentage_all_products", ("occurrencePercentage", "allProducts")),
8432
+ ("browse_node_metrics_occurrence_percentage_top_twenty_five_percent_products",
8433
+ ("occurrencePercentage", "topTwentyFivePercentProducts")),
8434
+ ("browse_node_metrics_star_rating_impact_all_products", ("starRatingImpact", "allProducts")),
8435
+ ("browse_node_metrics_star_rating_impact_top_twenty_five_percent_products",
8436
+ ("starRatingImpact", "topTwentyFivePercentProducts")),
8437
+ )
8438
+ BROWSE_NODE_TOPIC_TYPES = {
8439
+ **dict.fromkeys(("browse_node_id", "browse_node_name", "marketplace_id", "sort_by", "sentiment", "topic",
8440
+ "date_range_start_date", "date_range_end_date", "review_snippet_1", "review_snippet_2",
8441
+ "review_snippet_3", "our_asins"), "VARCHAR"),
8442
+ **dict.fromkeys((name for name, _path in BROWSE_NODE_TOPIC_METRICS), "DOUBLE"),
8443
+ **{name: kind for name, kind in _SUBTOPIC_TYPES.items() if "_number_of_mentions" not in name},
8444
+ "our_asin_count": "BIGINT",
8445
+ }
8446
+
8447
+
8448
+ def _browse_node_topic_evidence(records, nodes, marketplace):
8449
+ """Count raw category topics per node and sort and retain their metrics before the row parser runs."""
8450
+ sorts = _client().FEEDBACK_SORTS
8451
+ expected = {(node, sort) for node in nodes for sort in sorts}
8452
+ counts, metrics, absent, seen = {}, {}, [], set()
8453
+ metricless = 0
8454
+ for record in records:
8455
+ node, sort = str(record.get("browseNodeId")), record.get("_sort_by")
8456
+ if (node, sort) not in expected or (node, sort) in seen or record.get("marketplaceId") != marketplace:
8457
+ raise ValueError("Amazon Customer Feedback returned a missing, duplicate or foreign browse node or marketplace")
8458
+ seen.add((node, sort))
8459
+ no_content = record.get("_no_feedback_reason")
8460
+ topics = record.get("topics")
8461
+ if not no_content and not isinstance(topics, dict):
8462
+ raise ValueError("Amazon getBrowseNodeReviewTopics returned no valid topics object")
8463
+ total = 0
8464
+ for sentiment, key in (("positive", "positiveTopics"), ("negative", "negativeTopics")):
8465
+ group = None if no_content else topics.get(key)
8466
+ group = [] if group is None else group
8467
+ if not isinstance(group, list):
8468
+ raise ValueError("Amazon getBrowseNodeReviewTopics returned an invalid sentiment topic list")
8469
+ counts[json.dumps([node, sort, sentiment], separators=(",", ":"))] = len(group)
8470
+ total += len(group)
8471
+ for topic in group:
8472
+ if (not isinstance(topic, dict) or not isinstance(topic.get("topic"), str) or not topic["topic"].strip()
8473
+ or not isinstance(topic.get("browseNodeMetrics", {}), (dict, type(None)))):
8474
+ raise ValueError("Amazon category review topic requires a topic name and a browseNodeMetrics object or none")
8475
+ metricless += not isinstance(topic.get("browseNodeMetrics"), dict)
8476
+ identity = json.dumps([node, marketplace, sort, sentiment, topic["topic"]], separators=(",", ":"))
8477
+ if identity in metrics:
8478
+ raise ValueError("Amazon returned duplicate topics for one browse node and sentiment")
8479
+ values = []
8480
+ for _name, path in BROWSE_NODE_TOPIC_METRICS:
8481
+ value = topic.get("browseNodeMetrics")
8482
+ for segment in path:
8483
+ if value is not None and not isinstance(value, dict):
8484
+ raise ValueError("Amazon category review topic returned invalid nested metrics")
8485
+ value = value.get(segment) if value is not None else None
8486
+ values.append(_feedback_number(value))
8487
+ metrics[identity] = values
8488
+ if not total:
8489
+ absent.append({"browse_node_id": node, "sort_by": sort,
8490
+ "code": "NO_CONTENT" if no_content else "NO_TOPICS",
8491
+ "reason": no_content or "Amazon returned no positive or negative category topics for this browse node and marketplace."})
8492
+ if seen != expected:
8493
+ raise ValueError("Amazon Customer Feedback did not return a response for every browse node")
8494
+ return {"count": sum(counts.values()), "metricless": metricless, "source_counts": counts, "source_metrics": metrics,
8495
+ "marketplace_id": marketplace, "absent_browse_nodes": absent}
8496
+
8497
+
8498
+ def browse_node_topic_rows(records, document, our_asins):
8499
+ """Flatten each browse node's category topics with the tenant's own ASINs in that node."""
8500
+ rows = []
8501
+ for record in records:
8502
+ node = str(record["browseNodeId"])
8503
+ dates = record.get("dateRange") if isinstance(record.get("dateRange"), dict) else {}
8504
+ ours = sorted(our_asins.get(node, {}).get("asins", ()))
8505
+ for sentiment, key in (("positive", "positiveTopics"), ("negative", "negativeTopics")):
8506
+ for topic in (record.get("topics") or {}).get(key) or []:
8507
+ row = dict.fromkeys(BROWSE_NODE_TOPIC_TYPES)
8508
+ row.update(browse_node_id=node, marketplace_id=record["marketplaceId"], sort_by=record["_sort_by"],
8509
+ browse_node_name=record.get("displayName") or our_asins.get(node, {}).get("name"),
8510
+ sentiment=sentiment, topic=topic["topic"],
8511
+ date_range_start_date=str(dates.get("startDate") or "")[:10] or None,
8512
+ date_range_end_date=str(dates.get("endDate") or "")[:10] or None,
8513
+ our_asins=json.dumps(ours, separators=(",", ":")), our_asin_count=len(ours))
8514
+ for name, path in BROWSE_NODE_TOPIC_METRICS:
8515
+ value = topic.get("browseNodeMetrics")
8516
+ for segment in path:
8517
+ value = value.get(segment) if isinstance(value, dict) else None
8518
+ row[name] = value
8519
+ for index, snippet in enumerate(_feedback_snippets(topic.get("reviewSnippets")), 1):
8520
+ row[f"review_snippet_{index}"] = snippet
8521
+ row.update(_subtopic_columns(topic, mentions=False))
8522
+ rows.append(_origin(row, document, len(rows) + 1, "getBrowseNodeReviewTopics"))
8523
+ return rows
8524
+
8525
+
8526
+ def _sync_browse_node_review_topics(con, fields):
8527
+ """Replace the category topic snapshot of every browse node the active child ASINs sit in, both sorts."""
8528
+ asins = _feedback_asins(con)
8529
+ marketplace = fields.get("marketplace_id") or _client().US_MARKETPLACE_ID
8530
+ located = _client().item_browse_nodes(fields, asins)
8531
+ nodes, unplaced = {}, []
8532
+ for asin in asins:
8533
+ node = located.get(asin) or {}
8534
+ if node.get("browse_node_id"):
8535
+ entry = nodes.setdefault(node["browse_node_id"], {"name": node.get("browse_node_name"), "asins": set()})
8536
+ entry["asins"].add(asin)
8537
+ else:
8538
+ unplaced.append({"asin": asin, "code": "NO_BROWSE_NODE",
8539
+ "reason": node.get("_no_feedback_reason") or "Amazon returned no browse node for this ASIN."})
8540
+ records = _sorted_feedback(_client().browse_node_review_topics(fields, sorted(nodes)))
8541
+ unresolved = _null_unresolved_metrics(records)
8542
+ repeated = _drop_repeated_topics(records)
8543
+ evidence = _browse_node_topic_evidence(records, sorted(nodes), marketplace)
8544
+ rows = browse_node_topic_rows(records, "operation:" + uuid.uuid4().hex, nodes)
8545
+ period = datetime.now(timezone.utc).date().isoformat()
8546
+ for row in rows:
8547
+ row["_snapshot_date"] = period
8548
+ reason = (FEEDBACK_NO_ASINS if not asins else
8549
+ "Amazon returned no browse node for any active child ASIN. The per-ASIN reasons are recorded in the sync evidence."
8550
+ if not nodes else
8551
+ "Amazon Customer Feedback returned no category review topics for these browse nodes. "
8552
+ "The per-node reasons are recorded in the sync evidence. Amazon refreshes this data weekly.")
8553
+ marker = {"start": period, "end": period, "column": "_snapshot_date", **evidence, "absent_asins": unplaced,
8554
+ "unresolved_metrics": unresolved, "repeated_topics": repeated,
8555
+ "empty_reason": reason if not rows else ""}
8556
+ count = _replace_period(con, "browse_node_review_topics", rows,
8557
+ sorted(set(BROWSE_NODE_TOPIC_TYPES) | {key for row in rows for key in row}),
8558
+ "_snapshot_date", period, marker=marker, snapshot=True)
8559
+ _record_state(con, entity_key("limit_browse_node_review_topics"), "done", detail=json.dumps({
8560
+ "reason": marker["empty_reason"], "absent_asins": unplaced,
8561
+ "absent_browse_nodes": evidence["absent_browse_nodes"], "unresolved_metrics": unresolved, "repeated_topics": repeated}))
8562
  return count
8563
 
8564
 
 
8570
  return _sync_review_topics(con, fields)
8571
  if table == "review_trends":
8572
  return _sync_review_trends(con, fields)
8573
+ if table == "browse_node_review_topics":
8574
+ return _sync_browse_node_review_topics(con, fields)
8575
  if table == "economics":
8576
  start, end = _utc(w0).date().isoformat(), _utc(w1).date().isoformat()
8577
  if start >= end:
 
8981
  identity = json.dumps([row.get("asin"), row.get("marketplace_id"), row.get("sentiment"),
8982
  row.get("topic"), row.get("month")], separators=(",", ":"))
8983
  metrics[identity] = [row.get("date_range_start_date"), row.get("date_range_end_date"),
8984
+ *(row.get(name) for name, _path in _TREND_METRICS)]
8985
  total = con.execute(f"SELECT COUNT(*) FROM {table_name(table)}").fetchone()[0]
8986
  trends_match = (marker["count"] == len(rows) == len(metrics) == total
8987
  and counts == marker["source_counts"] and metrics == marker["source_metrics"])
 
8992
  counts = dict.fromkeys(marker["source_counts"], 0)
8993
  metrics = {}
8994
  for row in rows:
8995
+ group = json.dumps([row.get("asin"), row.get("sort_by"), row.get("sentiment")], separators=(",", ":"))
8996
  counts[group] = counts.get(group, 0) + 1
8997
+ identity = json.dumps([row.get("asin"), row.get("marketplace_id"), row.get("sort_by"),
8998
+ row.get("sentiment"), row.get("topic")], separators=(",", ":"))
8999
  metrics[identity] = [row.get(name) for name, _path in REVIEW_TOPIC_METRICS]
9000
  total = con.execute(f"SELECT COUNT(*) FROM {table_name(table)}").fetchone()[0]
9001
  matches = (marker["count"] == len(rows) == len(metrics) == total
 
9003
  return {"state": "ok" if matches else "refused", "expected": marker["count"], "actual": len(rows),
9004
  "reason": (marker["empty_reason"] if matches and not rows else
9005
  f"Amazon returned {marker['count']} review topics. Stored {len(rows)} topics with per-ASIN, sentiment and metric reconciliation.")}
9006
+ if table == "browse_node_review_topics":
9007
+ counts = dict.fromkeys(marker["source_counts"], 0)
9008
+ metrics = {}
9009
+ for row in rows:
9010
+ group = json.dumps([row.get("browse_node_id"), row.get("sort_by"), row.get("sentiment")],
9011
+ separators=(",", ":"))
9012
+ counts[group] = counts.get(group, 0) + 1
9013
+ identity = json.dumps([row.get("browse_node_id"), row.get("marketplace_id"), row.get("sort_by"),
9014
+ row.get("sentiment"), row.get("topic")], separators=(",", ":"))
9015
+ metrics[identity] = [row.get(name) for name, _path in BROWSE_NODE_TOPIC_METRICS]
9016
+ total = con.execute(f"SELECT COUNT(*) FROM {table_name(table)}").fetchone()[0]
9017
+ matches = (marker["count"] == len(rows) == len(metrics) == total
9018
+ and counts == marker["source_counts"] and metrics == marker["source_metrics"])
9019
+ return {"state": "ok" if matches else "refused", "expected": marker["count"], "actual": len(rows),
9020
+ "reason": (marker["empty_reason"] if matches and not rows else
9021
+ f"Amazon returned {marker['count']} category review topics. Stored {len(rows)} topics with per-node, sort, sentiment and metric reconciliation.")}
9022
  if table == "fee_estimates_api":
9023
  expected, actual = marker["count"], len(rows)
9024
  identities = {row.get("estimate_id") for row in rows}
 
9752
  # A watermark is written when a table's history is complete, so its next window is
9753
  # bounded; deferring it too froze every loaded table behind one refused report.
9754
  for table in want:
9755
+ # An entity snapshot owes no history, so the first load never holds it.
9756
  if ((read_watermark(con, table) and (not refresh_loaded or table not in repairs))
9757
+ or table in completed or table in FEEDBACK_TABLES
9758
  or (table == "settlements" and not refresh_loaded)):
9759
  continue
9760
  report["tables"][table] = {"state": "deferred", "reason":
 
9767
  _record_state(con, entity_key("sync_tick_" + table), "deferred", tick.get("rows") or 0, tick.get("detail"))
9768
  want = [table for table in want if (read_watermark(con, table)
9769
  and (not refresh_loaded or table not in repairs))
9770
+ or table in completed or table in FEEDBACK_TABLES
9771
  or (table == "settlements" and not refresh_loaded)]
9772
  if scheduled and not first_load:
9773
  ticks = {table: _read_state(con, entity_key("sync_tick_" + table)) for table in TABLES}
 
9795
  elif not first_load:
9796
  # The vendor's report floor also applies to an operator's Sync now click.
9797
  for table in list(want):
9798
+ # A Customer Feedback refusal stored nothing, so its weekly floor counts from a pull that landed.
9799
+ if (table in FEEDBACK_TABLES
9800
+ and (_read_state(con, entity_key(table)) or {}).get("state") in ("refused", "failed")):
9801
+ continue
9802
  reason = _interval_reason(table, _read_state(con, entity_key("sync_tick_" + table)),
9803
  datetime.now(timezone.utc))
9804
  if reason:
 
10909
  out[table] = {"state": "unproven", "reason": "table has never synced",
10910
  "reconciles": spec.get("validates_against")}
10911
  continue
10912
+ if table in ("fee_estimates_api", "replenishment", "inbound_shipments", *FEEDBACK_TABLES):
10913
  out[table] = {"rows": n, "reconciles": spec.get("validates_against"),
10914
  **_validate_margin(con, table)}
10915
  continue
 
11595
  "numberOfMentions": generation[0] + 2, "occurrencePercentage": 12.5,
11596
  "starRatingImpact": 0.25}}], "negativeTopics": None}} for asin in asins]
11597
 
11598
+ def browse_nodes(_fields, asins):
11599
+ return {asin: {"browse_node_id": "1000", "browse_node_name": "Fixture node"} for asin in asins}
11600
+
11601
+ def review_topic_records(fields, asins):
11602
+ empty = [{"asin": asin, "marketplaceId": cx.US_MARKETPLACE_ID, "topics": {"positiveTopics": None}}
11603
+ for asin in asins]
11604
+ return {"MENTIONS": feedback_records(fields, asins), "STAR_RATING_IMPACT": empty,
11605
+ "browse_nodes": browse_nodes(fields, asins)}
11606
+
11607
+ def category_records(_fields, nodes):
11608
+ calls.append(("browse_node_review_topics", {}))
11609
+ topics = [{"topic": f"Fixture category {generation[0]} {i}", "browseNodeMetrics": {
11610
+ "occurrencePercentage": {"allProducts": 12.5}}} for i in range(size("browse_node_review_topics"))]
11611
+ return {"MENTIONS": [{"browseNodeId": node, "marketplaceId": cx.US_MARKETPLACE_ID,
11612
+ "topics": {"positiveTopics": topics}} for node in nodes],
11613
+ "STAR_RATING_IMPACT": [{"browseNodeId": node, "marketplaceId": cx.US_MARKETPLACE_ID, "topics": {}}
11614
+ for node in nodes]}
11615
+
11616
  def snapshot_counts(requests):
11617
  return Counter(table for table, _kwargs in requests if table in snapshots)
11618
 
 
11652
  patch.object(cx, "outbound_fulfillment_orders", side_effect=outbound_records),
11653
  patch.object(cx, "inbound_shipment_items", side_effect=inbound_records),
11654
  patch.object(cx, "item_review_feedback", side_effect=feedback_records),
11655
+ patch.object(cx, "item_review_topics", side_effect=review_topic_records),
11656
+ patch.object(cx, "item_browse_nodes", side_effect=browse_nodes),
11657
+ patch.object(cx, "browse_node_review_topics", side_effect=category_records),
11658
  patch.object(cx, "replenishment_metrics", side_effect=replenishment_records),
11659
  patch.object(cx, "awd_inventory", return_value=[]),
11660
  patch.object(cx, "access_token", side_effect=AssertionError("offline snapshot sync fixture")),
 
13147
  "occurrencePercentage": 12.5, "starRatingImpact": 0.2}}],
13148
  "negativeTopics": [{"topic": "Fixture assembly", "asinMetrics": {"numberOfMentions": 2,
13149
  "occurrencePercentage": 6.25, "starRatingImpact": -0.1}}]}}]
13150
+ node = {"browse_node_id": "3734491", "browse_node_name": "Fixture node"}
13151
+ review_topics = {**{sort: feedback for sort in cx.FEEDBACK_SORTS}, "browse_nodes": {"B000000001": node}}
13152
+ category = {sort: [{"browseNodeId": "3734491", "displayName": "Fixture node", "marketplaceId": cx.US_MARKETPLACE_ID,
13153
+ "dateRange": {"startDate": "2026-05-06T00:00:00.000Z", "endDate": "2026-11-06T00:00:00.000Z"},
13154
+ "topics": {"positiveTopics": [{"topic": "Fixture glow", "browseNodeMetrics": {
13155
+ "occurrencePercentage": {"allProducts": 18.15, "topTwentyFivePercentProducts": 22.56},
13156
+ "starRatingImpact": {"allProducts": 1.58, "topTwentyFivePercentProducts": 1.15}}}],
13157
+ "negativeTopics": [{"topic": "Fixture size", "browseNodeMetrics": {
13158
+ "occurrencePercentage": {"allProducts": 14.31}}}]}}]
13159
+ for sort in cx.FEEDBACK_SORTS}
13160
 
13161
  try:
13162
  with patch.object(cx, "access_token", side_effect=AssertionError("No LWA in fixtures")), \
 
13167
  patch.object(cx, "inbound_shipment_items", return_value=inbound), \
13168
  patch.object(cx, "replenishment_metrics", return_value=replenishment), \
13169
  patch.object(cx, "item_review_feedback", return_value=feedback), \
13170
+ patch.object(cx, "item_review_topics", return_value=review_topics), \
13171
+ patch.object(cx, "item_browse_nodes", return_value={"B000000001": node}), \
13172
+ patch.object(cx, "browse_node_review_topics", return_value=category), \
13173
  patch.object(module, "_fee_estimate_inputs", return_value=[fee_input]), \
13174
  patch.object(cx, "fee_estimates", side_effect=lambda _f, estimates: _fixture_fee_records(estimates)):
13175
  for table in GROUP_A_TABLES:
 
13400
  finally:
13401
  con.close()
13402
 
13403
+ # A scheduled pass during an unfinished first load still runs a Customer Feedback snapshot.
13404
+ def feedback_pass(exempt):
13405
+ con = duckdb.connect(":memory:")
13406
+ con.execute("CREATE TABLE _sync_state (entity VARCHAR PRIMARY KEY, phase VARCHAR, last_id BIGINT, "
13407
+ "cursor_wd VARCHAR, rows BIGINT, updated_at VARCHAR)")
13408
+ rt = _FixtureSyncRuntime("group-a-feedback-fixture", ("review_topics",), "group-a-account")
13409
+ asked = []
13410
+ try:
13411
+ with _fixture_sync_store(con, rt), \
13412
+ patch.object(module, "datetime", Clock), \
13413
+ patch.object(module, "FEEDBACK_TABLES", FEEDBACK_TABLES if exempt else ()), \
13414
+ patch.object(module, "credential", return_value={"fixture": True}), \
13415
+ patch.object(module, "retention_check", return_value={"earliestTriggerAt": None}), \
13416
+ patch.object(module, "_sync_event"), \
13417
+ patch.object(cx, "item_review_topics", side_effect=lambda _f, asins: asked.append(asins) or {
13418
+ **{sort: [] for sort in cx.FEEDBACK_SORTS}, "browse_nodes": {}}), \
13419
+ patch.object(cx, "access_token", side_effect=AssertionError("No LWA in sync fixtures")), \
13420
+ patch.object(cx.urllib.request, "urlopen", side_effect=AssertionError("No network in sync fixtures")):
13421
+ with _fixture_account_state(con, rt, write=True) as account:
13422
+ record_watermark(account, "orders", end)
13423
+ _record_state(account, entity_key(FIRST_LOAD_END_KEY), "running", detail=_iso(end))
13424
+ result = sync(rt.key, tables=("review_topics",), rt=rt, log=lambda *_: None, scheduled=True)
13425
+ with _fixture_account_state(con, rt) as account:
13426
+ tick = _read_state(account, entity_key("sync_tick_review_topics"))
13427
+ return asked, (result.get("tables") or {}).get("review_topics") or {}, tick
13428
+ finally:
13429
+ con.close()
13430
+ asked, row, tick = feedback_pass(True)
13431
+ red_asked, red_row, red_tick = feedback_pass(False)
13432
+ ck("a scheduled pass during an unfinished first load runs review topics and records its tick",
13433
+ asked == [[]] and (tick or {}).get("state") == "done" and row.get("state") != "deferred"
13434
+ and not red_asked and not red_tick and red_row.get("state") == "deferred"
13435
+ and "first load" in red_row.get("reason", ""))
13436
+
13437
+ # A Sync now click retries a refused Customer Feedback table inside its weekly floor.
13438
+ def click_after(landed):
13439
+ con = duckdb.connect(":memory:")
13440
+ con.execute("CREATE TABLE _sync_state (entity VARCHAR PRIMARY KEY, phase VARCHAR, last_id BIGINT, "
13441
+ "cursor_wd VARCHAR, rows BIGINT, updated_at VARCHAR)")
13442
+ rt = _FixtureSyncRuntime("group-a-feedback-click", ("review_topics",), "group-a-account")
13443
+ asked = []
13444
+ try:
13445
+ with _fixture_sync_store(con, rt), \
13446
+ patch.object(module, "datetime", Clock), \
13447
+ patch.object(module, "credential", return_value={"fixture": True}), \
13448
+ patch.object(module, "retention_check", return_value={"earliestTriggerAt": None}), \
13449
+ patch.object(module, "_sync_event"), \
13450
+ patch.object(cx, "item_review_topics", side_effect=lambda _f, asins: asked.append(asins) or {
13451
+ **{sort: [] for sort in cx.FEEDBACK_SORTS}, "browse_nodes": {}}), \
13452
+ patch.object(cx, "access_token", side_effect=AssertionError("No LWA in sync fixtures")), \
13453
+ patch.object(cx.urllib.request, "urlopen", side_effect=AssertionError("No network in sync fixtures")):
13454
+ with _fixture_account_state(con, rt, write=True) as account:
13455
+ record_watermark(account, "orders", end)
13456
+ _record_state(account, entity_key(FIRST_LOAD_END_KEY), "done", detail=_iso(end))
13457
+ _record_state(account, entity_key("sync_tick_review_topics"), "done")
13458
+ _record_state(account, entity_key("review_topics"), landed, 0, "fixture landed state")
13459
+ result = sync(rt.key, tables=("review_topics",), rt=rt, log=lambda *_: None)
13460
+ return asked, (result.get("tables") or {}).get("review_topics") or {}
13461
+ finally:
13462
+ con.close()
13463
+ retried, retried_row = click_after("refused")
13464
+ held, held_row = click_after("done")
13465
+ ck("Sync now retries a refused review table inside its weekly floor and defers a landed one",
13466
+ retried == [[]] and retried_row.get("state") != "deferred"
13467
+ and not held and held_row.get("state") == "deferred")
13468
+
13469
  # Client pagination: an empty middle page must not end the traversal.
13470
  for operation, interval in cx.ENTITY_INTERVALS.items():
13471
  if operation == "getMyFeesEstimates":
platform/modules/product_data.py CHANGED
@@ -741,7 +741,45 @@ def _prune_product_overlay_bounded(source, generation):
741
 
742
 
743
  def source_batches(cur, *, st, team_id=None, ids=None, needed=None, raw_fields=()):
744
- """Stream catalogue identities, refuse global hash collisions and project bounded code batches."""
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
745
  from harness.canonical_source import bounded, BATCH
746
  with bounded(cur, 'identity', [], st) as identity:
747
  def codes():
@@ -762,11 +800,9 @@ def source_batches(cur, *, st, team_id=None, ids=None, needed=None, raw_fields=(
762
  f'FROM "{raw.table}" GROUP BY pid HAVING count(DISTINCT json_extract_string(body, \'/code\')) > 1 LIMIT 1').fetchone()
763
  if collision:
764
  raise ValueError(f'product_data: pid collision at {collision[0]}. Refusing the catalogue.')
765
- selected = '' if ids is None else " WHERE try_cast(json_extract_string(body, '/pid') AS BIGINT) IN (SELECT unnest(?::BIGINT[]))"
766
- params = [] if ids is None else [list(ids)]
767
  population = identity.snapshot(())
768
  cur.execute(f'INSERT INTO "{population.table}" SELECT min(ordinal), body FROM "{raw.table}"'
769
- f'{selected} GROUP BY body', params)
770
  # Keep the tracked snapshot name while materialising its sorted, typed seek key.
771
  cur.execute(f'CREATE OR REPLACE TEMP TABLE "{population.table}" AS '
772
  f"SELECT ordinal, body, try_cast(json_extract_string(body, '/pid') AS BIGINT) pid "
@@ -778,14 +814,9 @@ def source_batches(cur, *, st, team_id=None, ids=None, needed=None, raw_fields=(
778
  'WHERE pid > ? ORDER BY pid LIMIT ?', [last, BATCH]).fetchall()
779
  if not codes:
780
  return
781
- with bounded(cur, 'product', [code for _, code in codes], st):
782
- rows = ([{'pid': pid, 'code': code} for pid, code in codes]
783
- if needed is not None and not needed else
784
- pool(team_id=team_id, raw_fields=raw_fields, needed=needed))
785
- yield rows
786
  last = codes[-1][0]
787
- rows.clear()
788
- codes.clear()
789
 
790
 
791
  def _fba_by_code(cat):
 
741
 
742
 
743
  def source_batches(cur, *, st, team_id=None, ids=None, needed=None, raw_fields=()):
744
+ """Project bounded code batches: the whole active catalogue, or only the page's `ids`."""
745
+ from contextlib import closing
746
+ from harness.canonical_source import bounded
747
+ with closing(_catalogue_codes(cur, st) if ids is None else _page_codes(cur, ids)) as batches:
748
+ for codes in batches:
749
+ with bounded(cur, 'product', [code for _, code in codes], st):
750
+ rows = ([{'pid': pid, 'code': code} for pid, code in codes]
751
+ if needed is not None and not needed else
752
+ pool(team_id=team_id, raw_fields=raw_fields, needed=needed))
753
+ yield rows
754
+ rows.clear()
755
+ codes.clear()
756
+
757
+
758
+ def _page_codes(cur, ids):
759
+ """`[(pid, code)]` batches for the active products whose pid is in `ids`, sorted by pid.
760
+
761
+ A pid is a crc32 the mirror cannot compute, so each active code is hashed once and only the
762
+ selected ones are kept. The collision refusal covers the selected pids only.
763
+ """
764
+ from harness.canonical_source import BATCH
765
+ wanted, found = {int(pid) for pid in ids}, {}
766
+ result = cur.execute('SELECT id, default_code FROM product_product WHERE active = TRUE')
767
+ while True:
768
+ page = result.fetchmany(BATCH)
769
+ if not page:
770
+ break
771
+ for product_id, default_code in page:
772
+ code = str(default_code).strip() if default_code else f"pid:{product_id}"
773
+ pid = sku_pid(code)
774
+ if pid in wanted and found.setdefault(pid, code) != code:
775
+ raise ValueError(f'product_data: pid collision at {pid}. Refusing the catalogue.')
776
+ selected = sorted(found.items())
777
+ for start in range(0, len(selected), BATCH):
778
+ yield selected[start:start + BATCH]
779
+
780
+
781
+ def _catalogue_codes(cur, st):
782
+ """`[(pid, code)]` batches over every active product, refusing a pid collision anywhere."""
783
  from harness.canonical_source import bounded, BATCH
784
  with bounded(cur, 'identity', [], st) as identity:
785
  def codes():
 
800
  f'FROM "{raw.table}" GROUP BY pid HAVING count(DISTINCT json_extract_string(body, \'/code\')) > 1 LIMIT 1').fetchone()
801
  if collision:
802
  raise ValueError(f'product_data: pid collision at {collision[0]}. Refusing the catalogue.')
 
 
803
  population = identity.snapshot(())
804
  cur.execute(f'INSERT INTO "{population.table}" SELECT min(ordinal), body FROM "{raw.table}"'
805
+ ' GROUP BY body')
806
  # Keep the tracked snapshot name while materialising its sorted, typed seek key.
807
  cur.execute(f'CREATE OR REPLACE TEMP TABLE "{population.table}" AS '
808
  f"SELECT ordinal, body, try_cast(json_extract_string(body, '/pid') AS BIGINT) pid "
 
814
  'WHERE pid > ? ORDER BY pid LIMIT ?', [last, BATCH]).fetchall()
815
  if not codes:
816
  return
817
+ # The consumer clears each batch, so the seek key is read before it is lent.
 
 
 
 
818
  last = codes[-1][0]
819
+ yield codes
 
820
 
821
 
822
  def _fba_by_code(cat):
web/public/AIOS_DEPLOY_RECEIPT CHANGED
@@ -1 +1 @@
1
- fc20fb27640c854759ec5061266f0c3a70ad8f7831d67eeb401f0b2125c01625
 
1
+ 272331cb0e6d8111127b0284f29e5196f1b7043e895308d2b1c6612feef0ce08
web/src/connectors/ConnectorsPage.tsx CHANGED
@@ -263,12 +263,23 @@ export interface AmazonTableSync {
263
  lastSyncAt: string | null;
264
  nextSyncAt: string;
265
  perDay: number;
 
 
266
  state: string;
267
  reason?: string;
268
  /** A source-internal mismatch: the table published, and both counts are named. */
269
  warning?: string;
270
  }
271
 
 
 
 
 
 
 
 
 
 
272
  export interface AmazonSyncScheduleReading {
273
  state: "ready" | "empty" | "unavailable";
274
  timezone: string;
@@ -295,7 +306,9 @@ export function parseAmazonSyncSchedule(body: Record<string, unknown>): AmazonSy
295
  || typeof row.account !== "string" || typeof row.state !== "string"
296
  || (row.lastSyncAt !== null && (typeof row.lastSyncAt !== "string" || !Number.isFinite(Date.parse(row.lastSyncAt))))
297
  || typeof row.nextSyncAt !== "string" || !Number.isFinite(Date.parse(row.nextSyncAt))
298
- || typeof row.perDay !== "number" || !Number.isInteger(row.perDay) || row.perDay < 1) return unavailable;
 
 
299
  const identity = JSON.stringify([row.connectionId, row.table]);
300
  if (identities.has(identity)) return unavailable;
301
  identities.add(identity);
@@ -336,7 +349,7 @@ export function AmazonSyncSchedule({ reading, problem = "", onRetry }: {
336
  <table className="conn-settings-databases" aria-label="Amazon table sync schedule">
337
  <thead><tr><th scope="col">Table</th>{multipleAccounts ? <th scope="col">Account</th> : null}
338
  <th scope="col">Last sync</th><th scope="col">Next sync</th>
339
- <th scope="col" className="conn-settings-number">Passes per day</th></tr></thead>
340
  <tbody>{reading.tables.map((row) => <tr key={JSON.stringify([row.connectionId, row.table])}
341
  data-amazon-table={row.table} data-connection-id={row.connectionId} data-table-sync-state={row.state}>
342
  <th scope="row">{row.table.replace(/_/g, " ")}</th>
@@ -351,7 +364,7 @@ export function AmazonSyncSchedule({ reading, problem = "", onRetry }: {
351
  : row.state === "throttled" ? <p role="status">Rate limited, retry scheduled</p>
352
  : ["failed", "refused"].includes(row.state) ? <p role="status">Sync needs attention</p> : null}
353
  </td>
354
- <td className="conn-settings-number">{row.perDay}</td>
355
  </tr>)}</tbody>
356
  </table>
357
  </div>
 
263
  lastSyncAt: string | null;
264
  nextSyncAt: string;
265
  perDay: number;
266
+ /** The CADENCE period. A row without one refreshes daily. */
267
+ period?: AmazonRefreshPeriod;
268
  state: string;
269
  reason?: string;
270
  /** A source-internal mismatch: the table published, and both counts are named. */
271
  warning?: string;
272
  }
273
 
274
+ export type AmazonRefreshPeriod = "day" | "week" | "month";
275
+
276
+ /** One plain word for how often a table refreshes. */
277
+ export function amazonRefreshLabel(row: Pick<AmazonTableSync, "perDay" | "period">): string {
278
+ if (row.period === "week") return "Weekly";
279
+ if (row.period === "month") return "Monthly";
280
+ return row.perDay > 1 ? `${row.perDay} times a day` : "Daily";
281
+ }
282
+
283
  export interface AmazonSyncScheduleReading {
284
  state: "ready" | "empty" | "unavailable";
285
  timezone: string;
 
306
  || typeof row.account !== "string" || typeof row.state !== "string"
307
  || (row.lastSyncAt !== null && (typeof row.lastSyncAt !== "string" || !Number.isFinite(Date.parse(row.lastSyncAt))))
308
  || typeof row.nextSyncAt !== "string" || !Number.isFinite(Date.parse(row.nextSyncAt))
309
+ || typeof row.perDay !== "number" || !Number.isInteger(row.perDay) || row.perDay < 1
310
+ || (row.period !== undefined
311
+ && (typeof row.period !== "string" || !["day", "week", "month"].includes(row.period)))) return unavailable;
312
  const identity = JSON.stringify([row.connectionId, row.table]);
313
  if (identities.has(identity)) return unavailable;
314
  identities.add(identity);
 
349
  <table className="conn-settings-databases" aria-label="Amazon table sync schedule">
350
  <thead><tr><th scope="col">Table</th>{multipleAccounts ? <th scope="col">Account</th> : null}
351
  <th scope="col">Last sync</th><th scope="col">Next sync</th>
352
+ <th scope="col">Refresh</th></tr></thead>
353
  <tbody>{reading.tables.map((row) => <tr key={JSON.stringify([row.connectionId, row.table])}
354
  data-amazon-table={row.table} data-connection-id={row.connectionId} data-table-sync-state={row.state}>
355
  <th scope="row">{row.table.replace(/_/g, " ")}</th>
 
364
  : row.state === "throttled" ? <p role="status">Rate limited, retry scheduled</p>
365
  : ["failed", "refused"].includes(row.state) ? <p role="status">Sync needs attention</p> : null}
366
  </td>
367
+ <td data-amazon-refresh={row.period ?? "day"}>{amazonRefreshLabel(row)}</td>
368
  </tr>)}</tbody>
369
  </table>
370
  </div>