diff --git a/RELEASES.json b/RELEASES.json
index 73b8bc73b9009e2f53f151a0afc50613868a43e9..9d952586a452115493bc1e2944b380c5dd76b993 100644
--- a/RELEASES.json
+++ b/RELEASES.json
@@ -1,4 +1,4 @@
{
- "current": "8538d73390",
+ "current": "659eeb316",
"releases": []
}
\ No newline at end of file
diff --git a/VERSION b/VERSION
index c894f9f63a84992e48427aeb7280da7cb72e1e66..0b3e37cdeeb3a48b4d6d7b8ea88c65d00248a38c 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-8538d73390
+659eeb316
diff --git a/api/deps.py b/api/deps.py
index f8419b82aff2c7451afb7d936bd659b3567ff041..c23bf3a7369f3731898aaa96c8a30e21a67b9ee1 100644
--- a/api/deps.py
+++ b/api/deps.py
@@ -1809,17 +1809,62 @@ def _touch_active(uname):
pass
+def refuse_view_as_write(request: Request):
+ """Refuse every view-as method except GET and HEAD before any write."""
+ if "X-View-As" in request.headers and request.method not in {"GET", "HEAD"}:
+ raise err(403, "view_as_read_only", "Viewing as a member is read-only.")
+
+
+def _view_as_refused():
+ return err(403, "view_as_refused", "This member view is unavailable.")
+
+
+def _view_as_user(request, actor, tenant, rt):
+ """Resolve an active member of the actor's tenant and audit this request once."""
+ if "X-View-As" not in request.headers:
+ return actor
+ if not perms.is_admin(actor):
+ raise _view_as_refused()
+ started_at = datetime.now(timezone.utc).isoformat()
+ target = request.headers["X-View-As"].strip().lower()
+ if not target:
+ raise _view_as_refused()
+ rec = copy.deepcopy(_session_registry(target).get(target))
+ if (not rec or not rec.get("active", True) or rec.get("role", "user") != "user"
+ or str(rec.get("tenant") or "").strip().lower() != tenant):
+ raise _view_as_refused()
+ member = users._public(target, rec)
+ audit_key = (tenant, actor["username"], target)
+ if getattr(request.state, "_view_as_audited", None) != audit_key:
+ from core import activity_log
+ activity_log.append(
+ rt, kind="view_as", object={"app": "settings", "id": target},
+ by=actor["username"], on_behalf_of=target,
+ changes={"member": {"before": None, "after": target},
+ "started_at": {"before": None, "after": started_at}})
+ request.state._view_as_audited = audit_key
+ return member
+
+
def require_session(request: Request, response: Response) -> Session:
"""Verify the cookie, resolve the tenant, and REISSUE the cookie (the 8h idle window is
refreshed on use; the 30-day absolute expiry is carried over, never extended)."""
+ refuse_view_as_write(request)
+ viewing_as = "X-View-As" in request.headers
raw = request.cookies.get(aios_session.COOKIE_NAME)
if not raw:
+ if viewing_as:
+ raise _view_as_refused()
raise err(401, "no_session", "sign in to continue")
claims = aios_session.read(raw)
if not claims:
+ if viewing_as:
+ raise _view_as_refused()
raise err(401, "invalid_session", "Your session ended. Sign in again.")
user = _user_for(claims)
if not user:
+ if viewing_as:
+ raise _view_as_refused()
raise err(401, "invalid_session", "Your session ended. Sign in again.")
identity = {_TENANT_HEADER: str(claims["t"]),
_PRINCIPAL_HEADER: _principal(claims["t"], user["username"])}
@@ -1835,8 +1880,12 @@ def require_session(request: Request, response: Response) -> Session:
except KeyError:
# An unknown tenant slug is a 401, not a 404: the cookie is not a valid identity for this
# deployment, and confirming which slugs exist would answer a question we were not asked.
+ if viewing_as:
+ raise _view_as_refused()
raise err(401, "invalid_session", "Your session ended. Sign in again.")
- fresh, _ = aios_session.mint(claims["t"], user["username"], int(user.get("epoch") or 0),
+ actor = user
+ user = _view_as_user(request, actor, claims["t"], rt)
+ fresh, _ = aios_session.mint(claims["t"], actor["username"], int(actor.get("epoch") or 0),
absolute_expiry=claims["x"])
aios_session.set_cookie(response, request, fresh)
# ⭐ STAMP THE TENANT THIS ANSWER WAS SERVED FOR (owner report 2026-08-09: "I can see
@@ -1857,11 +1906,13 @@ def require_session(request: Request, response: Response) -> Session:
# `session.tenant` read back from something the request could influence.
response.headers[_TENANT_HEADER] = str(claims["t"])
response.headers[_PRINCIPAL_HEADER] = _principal(claims["t"], user["username"])
- # Observe only AFTER authentication admission. Read methods stay store-read-only; only an
- # admitted mutation can launch the persisted stamp, without moving it outside request timing.
+ if viewing_as:
+ response.headers["X-Viewing-As"] = user["username"]
+ # Ordinary reads only observe activity. View-as reads write their audit without
+ # changing either person's activity stamps.
# qa-runner has no global account record. Never let the normal activity telemetry create or
# touch one through the default tenant-#0 users store.
- if (not user.get("qa_runner")
+ if (not viewing_as and not user.get("qa_runner")
and not (request.method == "GET" and request.url.path == "/api/v1/notifications")):
_observe_active(claims["t"], user.get("username"))
if request.method not in {"GET", "HEAD", "OPTIONS"}:
@@ -1877,6 +1928,8 @@ def require_changes_session(request: Request, response: Response) -> Session | N
verified signed cookie is enough to return that no-data response; resolving the account record
would itself turn those legacy requests into a permanent Neon keepalive.
"""
+ if "X-View-As" in request.headers:
+ return require_session(request, response)
if store.backend() != "pg":
return require_session(request, response)
if request.headers.get("x-aios-change-check") == "focus-v1":
@@ -1918,6 +1971,9 @@ def require_session_or_bearer(request: Request, response: Response) -> Session:
`require_session` stays cookie only. A bearer session never reissues the cookie.
"""
+ refuse_view_as_write(request)
+ if "X-View-As" in request.headers:
+ return _cookie_session(request, response)
bearer = _bearer_header(request)
if bearer is None or request.cookies.get(aios_session.COOKIE_NAME):
try:
diff --git a/api/loopy_catalog_tools.py b/api/loopy_catalog_tools.py
index 3669273adc7e2c28a0f212b79d77bea42242c1d2..93548b00b498339bb2545a48a91bd812c98557e2 100644
--- a/api/loopy_catalog_tools.py
+++ b/api/loopy_catalog_tools.py
@@ -305,8 +305,10 @@ def catalog_pdf_within_budget(session, args, render=None, budget=None):
import contextvars
import sys
import threading
+ import time
import catalog_store
import app_templates
+ from core import request_timing
render = render or sys.modules[__name__].catalog_pdf
budget = export_budget() if budget is None else budget
@@ -329,8 +331,15 @@ def catalog_pdf_within_budget(session, args, render=None, budget=None):
import logging
logging.getLogger(__name__).warning("Catalog export result not saved (%s)", type(exc).__name__)
+ def run():
+ # HTTP completion cancels its timing state, so the export needs its own bounded scope.
+ with request_timing.request_scope() as measured:
+ measured.deadline = time.monotonic() + catalog_store.EXPORT_STALE_SECONDS
+ with request_timing.thread_scope():
+ work()
+
context = contextvars.copy_context()
- worker = threading.Thread(target=context.run, args=(work,), daemon=True, name="catalog-pdf")
+ worker = threading.Thread(target=context.run, args=(run,), daemon=True, name="catalog-pdf")
try:
worker.start()
except Exception:
diff --git a/api/main.py b/api/main.py
index b7c9e5b6559216345a7d434aa91b1b74bcafd5fe..840ea27c82d23e974a6a427fa4285993193f9510 100644
--- a/api/main.py
+++ b/api/main.py
@@ -1662,16 +1662,17 @@ def _sync_now_detail(provider, tenant, *, ok, detail="", cause=None):
return sentence + suffix
-def _run_sync_requests(outcome=None, *, during_odoo=False, hold_first_load=None):
- """Drain Sync now intents on the resident resync thread, the mirror's single writer.
+def _run_sync_requests(outcome=None, *, during_odoo=False, hold_first_load=None, amazon_only=False):
+ """Drain admitted Sync now intents before their resident projection.
- D-29 / intake F6: HTTP handlers only queue intents. Provider calls stay here, before the
- resident projection pass. A failed request cannot discard another tenant's queued work.
+ HTTP handlers queue intents. The slot loop owns Odoo and Meta, and its detached turn owns Amazon.
+ A failed request cannot discard another tenant's queued work.
`outcome`, when given, receives `{provider: "continuing" | "done" | "failed"}` so the loop can
project only what a desk wake actually changed.
"""
from core import connector_sync
- _drain_pending_amazon_revocations()
+ if amazon_only or globals().get("_AMAZON_DESK", {}).get("job") is None:
+ _drain_pending_amazon_revocations()
def progress_log(provider, tenant, receipt, rt):
"""Publish only authored progress shapes, with one bounded detail per running receipt."""
@@ -1772,7 +1773,11 @@ def _run_sync_requests(outcome=None, *, during_odoo=False, hold_first_load=None)
# A first-load continuation never runs inside an Odoo pass: it requeues itself without end.
hold = during_odoo if hold_first_load is None else hold_first_load
# Claim only work this turn executes. A later provider remains queued with an honest wait.
- pending = connector_sync.take_sync_requests(limit=1, exclude_providers=("odoo",) if during_odoo else (),
+ excluded = ({"odoo", "meta_ads"} if amazon_only else
+ ({"odoo"} if during_odoo else set()))
+ if not amazon_only and globals().get("_AMAZON_DESK", {}).get("managed"):
+ excluded.add("amazon_sp")
+ pending = connector_sync.take_sync_requests(limit=1, exclude_providers=excluded,
refresh_remote=True,
exclude_requested_by=("first load",) if hold else ())
while True:
@@ -2424,14 +2429,40 @@ def _rebuild_odoo_relational(why, previous_meta_fingerprint=None, meta_state_out
**({"cause": _raw_cause(key)} if _raw_cause(key) else {})}
for key, db_failure in blocking.items() if isinstance(db_failure, dict)]
if failed_databases is not None and table_keys is not None:
- # D-2679: a changed-tables pass keeps the failures of databases it did not run.
+ # Unassessed failures survive unless their exact missing-binding refusal is disproved.
try:
prior = (_rt.get(_rel.CONFIG_KEY) or {}).get("failedDatabases") or []
except Exception: # noqa: BLE001
prior = []
+ recovered_database_keys = set()
+ for item in prior:
+ if not isinstance(item, dict):
+ continue
+ key = item.get("key")
+ if (key in database_keys or key in database_failures
+ or item.get("code") != "materialization_failed"
+ or item.get("stage") != "materialization"
+ or item.get("cause") !=
+ "Refused: the database has no readable mirror binding"
+ or not _rel.is_read_through_key(key)):
+ continue
+ try:
+ _base, owner = _rel.split_table_key(key)
+ if owner is not None or not _ut.get_definition(key, st=_rt):
+ continue
+ if not any(_key == key and bucket in _rel.enabled_buckets(_rt)
+ for bucket, _key, _label, _fields in _rel.TABLES):
+ continue
+ with _rt.mirror_handle().ro_cursor() as cur:
+ nrows = routes_odoo_tables.population(key, cur=cur, rt=_rt)
+ if isinstance(nrows, int) and nrows >= 0:
+ recovered_database_keys.add(key)
+ except Exception: # noqa: BLE001
+ pass # A missing, filling or unreadable mirror cannot prove recovery.
failed_databases = [
item for item in prior if isinstance(item, dict)
- and item.get("key") not in database_keys] + failed_databases
+ and item.get("key") not in database_keys
+ and item.get("key") not in recovered_database_keys] + failed_databases
try:
_rel.record_sync_outcome(_rt, ok=not safe_errors, at=refreshed_at,
failed_databases=failed_databases)
@@ -3234,15 +3265,77 @@ def _sheet_warm_after_pass(tenant, good, generation_before, rebuilt):
def _sync_idle_wait(event, seconds):
"""The resync loop's wait. A `sync_worker.py --once` job exits here when it is idle (S82-T10)."""
+ if globals().get("_AMAZON_DESK", {}).get("job") is not None:
+ # An unclaimed or finishing desk turn still owns work, even between receipt states.
+ return event.wait(seconds)
import sync_runtime
return sync_runtime.idle_wait(event, seconds)
+def _amazon_intraday_pass(tenant_key, previous_fingerprint):
+ """Run due Amazon tables and their projection without advancing the Odoo pass clock."""
+ if globals().get("_AMAZON_DESK", {}).get("job") is not None:
+ return previous_fingerprint
+ try:
+ from harness import amazon_store, runtime
+ rt = runtime.get_runtime(tenant_key)
+ if not amazon_store.credential(tenant_key, rt=rt) or amazon_store.first_load_job_live(rt):
+ return previous_fingerprint
+ plan = amazon_store.scheduled_sync_plan(tenant_key, rt=rt)
+ if not plan["tables"]:
+ return previous_fingerprint
+ print("[aios-api] amazon intraday pass due: " + ", ".join(plan["tables"]), flush=True)
+ RESYNC_STATE["slotPass"] = False
+ _pull_amazon("intraday")
+ state = _rebuild_amazon_relational("intraday", rt, previous_fingerprint)
+ return state["fingerprint"] if state.get("known") else None
+ except Exception as exc:
+ print(f"[aios-api] amazon intraday pass failed ({type(exc).__name__}); "
+ "the next cadence check retries it", flush=True)
+ return None
+
+
+def _amazon_desk_pass(tenant, *, project=True):
+ """Run one Amazon desk claim off the slot clock and collect its projection after publication."""
+ import process_role
+ import time
+ if not process_role.may_write():
+ return False
+ from core import connector_sync
+ state = globals().setdefault("_AMAZON_DESK", {"managed": True, "job": None, "outcome": {}})
+ job = state["job"]
+ if job is not None:
+ if job.is_alive() or not project:
+ return True
+ job.join()
+ state["job"] = None
+ _project_desk(state["outcome"], tenant)
+ receipt = connector_sync.sync_request("amazon_sp", tenant)
+ if (not receipt or receipt["state"] != "queued" or receipt.get("requestedBy") == "first load"
+ or max(receipt.get("notBefore") or 0,
+ connector_sync.sync_not_before("amazon_sp", tenant) or 0) > time.time()):
+ return False
+ outcome = state["outcome"] = {}
+
+ def drain():
+ try:
+ _run_sync_requests(outcome, hold_first_load=True, amazon_only=True)
+ except Exception as exc:
+ print(f"[aios-api] amazon desk turn failed ({type(exc).__name__})", flush=True)
+ finally:
+ connector_sync.sync_wake_event().set()
+
+ # This turn inherits store-resync's heavy admission, including tenant-worker scope.
+ state["job"] = process_role.start("store-resync", "heavy", drain)
+ return state["job"] is not None
+
+
def _store_resync_loop():
"""Keep the analytical mirror current — the API-process stand-in for the re-sync the
Streamlit app piggybacks on page renders. Measure memos key on the pool stamp, so a
refreshed pool re-reads the freshly synced store."""
- if _freeze_holds("store resync"):
+ import process_role as _loop_role
+ if not _loop_role.may_write() or _freeze_holds("store resync"):
return
import time as _t
from core import connector_sync
@@ -3278,6 +3371,7 @@ def _store_resync_loop():
last_full_pass_at = _t.time()
# The least time between two desk-only passes of an Amazon first-load continuation.
desk_min_spacing_seconds = 30.0
+ amazon_check_seconds = 60.0
# H73: the start of the last daily-slot pass, and whether the boot restore already ran.
last_slot_pass_at = _memory.get("slotPassAt")
slot_restored = False
@@ -3368,7 +3462,7 @@ def _store_resync_loop():
_slot = (globals().get("_resolve_resync_slot") or (lambda: None))()
def _near_slot():
- # A first-load continuation does not start this close to the next Odoo slot.
+ # Completed desk projections wait inside the existing first-load slot quiet window.
if _slot is None:
return False
import odoo_relational as _rel_near
@@ -3383,6 +3477,7 @@ def _store_resync_loop():
# END drifted later every day and past the 26-hour stale line).
_woken = bool(_idle_wait(connector_sync.sync_wake_event(), min(
max(0.0, _every - (_t.time() - last_full_pass_at)),
+ amazon_check_seconds,
float("inf") if backfill_next_at is None else max(0.0, backfill_next_at - _t.time()))))
connector_sync.sync_wake_event().clear()
_slot_due = False
@@ -3396,6 +3491,7 @@ def _store_resync_loop():
_woken = bool(_idle_wait(connector_sync.sync_wake_event(), min(
0.0 if _rel_slot.slot_due(_slot_now, _slot, last_slot_pass_at)
else max(0.0, _rel_slot.slot_after(_slot_now, _slot) + 1.0 - _slot_now),
+ amazon_check_seconds,
float("inf") if backfill_next_at is None else max(0.0, backfill_next_at - _slot_now))))
connector_sync.sync_wake_event().clear()
_slot_now = _t.time()
@@ -3428,6 +3524,11 @@ def _store_resync_loop():
f"{_slot['tz']} slot; cleared the slot_pass_interrupted failure")
else:
RESYNC_STATE.pop("nextPassAt", None)
+ _cadence_due = not passes or _t.time() - last_full_pass_at >= _every
+ _amazon_desk_busy = (_amazon_desk_pass(_cad["tenant"], project=not (_slot_due or _near_slot()))
+ if _slot is not None or globals().get("_AMAZON_DESK") else False)
+ if (not _slot_due if _slot is not None else not _cadence_due):
+ last_amzn_fingerprint = _amazon_intraday_pass(_cad["tenant"], last_amzn_fingerprint)
# ⛔⛔ HOTFIX 2026-09-29 — A RESTART INSIDE THE DAILY WINDOW IS NOT A SYNC. The owner
# capped syncing at once a day, and a deploy is not a reason to sync again. So the first
# pass after a boot that lands within one cadence of the last scheduled pass only catches
@@ -3470,13 +3571,16 @@ def _store_resync_loop():
else:
_project_desk(_desk, _cad["tenant"])
if not slot_restored:
- _pull_amazon("boot", False)
+ if not _amazon_desk_busy:
+ _pull_amazon("boot", False)
# The restored mirror warms the workbooks once; Sheet warm reads only the mirror.
_sheet_warm_after_pass(_cad["tenant"], True, None, True)
slot_restored = True
print(f"[aios-api] store resync: no Odoo pass outside the daily slot; the next one "
f"runs at {RESYNC_STATE['nextPassAt']} ({_slot['at']} {_slot['tz']})")
continue
+ if (_slot is None and not _cadence_due and _odoo_requested is None and not _continuation):
+ continue
passes += 1
backfill_next_at = None
if not _continuation:
@@ -3723,9 +3827,10 @@ def _store_resync_loop():
except Exception as e: # noqa: BLE001
print(f"[aios-api] meta relational refresh failed (resync): "
f"{type(e).__name__}: {e}")
- if _resumed.get("amazon"):
+ # The independent desk retains Amazon's pull and projection while Odoo completes its pass.
+ if not _amazon_desk_busy and _resumed.get("amazon"):
_pull_amazon("boot", False)
- else:
+ elif not _amazon_desk_busy:
# The daily slot pass refreshes every Amazon table: a late extra pass yesterday must not defer it.
RESYNC_STATE["slotPass"] = _slot_claimed
try:
@@ -3739,7 +3844,8 @@ def _store_resync_loop():
try:
from harness import runtime as _runtime
_rt_resync = _runtime.get_runtime(_cad["tenant"])
- _amzn_state = _rebuild_amazon_relational("resync", _rt_resync, _amzn_previous)
+ _amzn_state = ({} if _amazon_desk_busy else
+ _rebuild_amazon_relational("resync", _rt_resync, _amzn_previous))
last_amzn_fingerprint = (_amzn_state["fingerprint"]
if _amzn_state.get("known") else None)
except Exception as e: # noqa: BLE001
diff --git a/api/routes_account_health.py b/api/routes_account_health.py
index e3a9e1772aa240aa498a6f890d17245e6ea3b50c..5dbace9bf5754b18a0e86e661166d77bcec09305 100644
--- a/api/routes_account_health.py
+++ b/api/routes_account_health.py
@@ -5,7 +5,8 @@ from urllib.parse import urlencode
from deps import Session, err, require_session
TEMPLATE_KIND = "account_health"
-router = APIRouter(prefix=f"/api/v1/app-kinds/{TEMPLATE_KIND}", tags=["account-health"])
+router = APIRouter(tags=["account-health"])
+app_router = APIRouter(prefix=f"/api/v1/app-kinds/{TEMPLATE_KIND}", tags=["account-health"])
SOURCE = "ut_amzn_account_health"
@@ -101,7 +102,7 @@ def owners(session):
return sorted((n for n in names if n), key=str.lower)
-@router.get("/summary")
+@app_router.get("/summary")
def read_summary(app: str = Query(..., min_length=1), search: str = Query(""),
offset: int = Query(0, ge=0), limit: int = Query(50, ge=1),
metric: str | None = Query(None), session: Session = Depends(_app_reader)):
@@ -115,7 +116,7 @@ def read_summary(app: str = Query(..., min_length=1), search: str = Query(""),
return {**out, "canEdit": _may_edit(session, _app(session, app))}
-@router.get("/history")
+@app_router.get("/history")
def read_history(metric: str = Query(..., min_length=1), cursor: str | None = Query(None),
limit: int = Query(400, ge=1), session: Session = Depends(_app_reader)):
ah = _module()
@@ -133,7 +134,7 @@ def read_history(metric: str = Query(..., min_length=1), cursor: str | None = Qu
return _call(ah.history, _tenant(session), metric, cursor=cursor, limit=limit, st=session.runtime)
-@router.get("/items")
+@app_router.get("/items")
def list_items(app: str = Query(..., min_length=1), status: str | None = Query(None),
search: str = Query(""), offset: int = Query(0, ge=0), metric: str | None = Query(None),
cursor: str | None = Query(None), limit: int = Query(50, ge=1),
@@ -149,7 +150,7 @@ def list_items(app: str = Query(..., min_length=1), status: str | None = Query(N
return {**listed, "owners": owners(session), "canEdit": _may_edit(session, _app(session, app))}
-@router.get("/items/{item_id}")
+@app_router.get("/items/{item_id}")
def read_item(item_id: str, app: str = Query(..., min_length=1),
session: Session = Depends(_app_reader)):
ah = _module()
@@ -163,7 +164,7 @@ def read_item(item_id: str, app: str = Query(..., min_length=1),
"canEdit": _may_edit(session, _app(session, app))}
-@router.patch("/items/{item_id}")
+@app_router.patch("/items/{item_id}")
def patch_item(item_id: str, owner: str | None = Body(None), status: str | None = Body(None),
due: str | None = Body(None), note: str | None = Body(None),
session: Session = Depends(_app_writer)):
@@ -176,13 +177,13 @@ def patch_item(item_id: str, owner: str | None = Body(None), status: str | None
due=due, note=note, st=session.runtime)
-@router.get("/settings")
+@app_router.get("/settings")
def read_settings(session: Session = Depends(_app_reader)):
ah = _module()
return _call(ah.settings, _tenant(session), st=session.runtime)
-@router.put("/settings")
+@app_router.put("/settings")
def write_settings(alertDistancePct: float | None = Body(None), defaultOwner: str | None = Body(None),
riskOverdueDays: float | None = Body(None),
session: Session = Depends(_app_writer)):
@@ -191,7 +192,7 @@ def write_settings(alertDistancePct: float | None = Body(None), defaultOwner: st
default_owner=defaultOwner, risk_overdue_days=riskOverdueDays, st=session.runtime)
-@router.get("/risks")
+@app_router.get("/risks")
def list_risks(app: str = Query(..., min_length=1), status: str | None = Query(None),
search: str = Query(""), offset: int = Query(0, ge=0), limit: int = Query(50, ge=1),
session: Session = Depends(_app_reader)):
@@ -205,7 +206,7 @@ def list_risks(app: str = Query(..., min_length=1), status: str | None = Query(N
return {**out, "owners": owners(session), "canEdit": _may_edit(session, _app(session, app))}
-@router.patch("/risks/{item_id}")
+@app_router.patch("/risks/{item_id}")
def patch_risk(item_id: str, owner: str | None = Body(None), status: str | None = Body(None),
caseId: str | None = Body(None), appealDate: str | None = Body(None),
outcome: str | None = Body(None), note: str | None = Body(None),
@@ -218,6 +219,22 @@ def patch_risk(item_id: str, owner: str | None = Body(None), status: str | None
case_id=caseId, appeal_date=appealDate, outcome=outcome, note=note, st=session.runtime)
+router.include_router(app_router)
+
+
+@router.get("/api/v1/account-health/categories/{key}")
+def read_category(key: str, session: Session = Depends(_app_reader)):
+ ah = _module()
+ context = _call(_narrowed, session)
+
+ def allowed(row, snapshot):
+ evidence = _call(ah.category_source, _tenant(session), session.runtime, row, snapshot)
+ return _source_allowed({"sourceEvidence": evidence}, context)
+
+ return _call(ah.category_detail, _tenant(session), key, user=session.user,
+ allowed=allowed if context is not None else None, st=session.runtime)
+
+
# --------------------------------------------------------------------------- R62 source narrowing
def _narrowed(session):
diff --git a/api/routes_admin.py b/api/routes_admin.py
index e3bad854d197f3f56cea6eb55ed419717197de18..068bf7fc9406d6c0d80f09fdb4a6af4a32ac1b33 100644
--- a/api/routes_admin.py
+++ b/api/routes_admin.py
@@ -143,17 +143,7 @@ def _view(uname, rec, governed=None, st=None, surface_keys=None):
# editor will open, so the two cannot disagree.
"accessSummary": _access_summary(rec, uname=uname, governed=governed, st=st,
surface_keys=surface_keys),
- "perms_v": int(rec.get("perms_v") or 0),
- # S79-T63: only a copied test account carries `test`, so a real account's keys stay exact.
- **({"test": _test_marker(rec)} if _test_marker(rec) else {})}
-
-
-def _test_marker(rec):
- mark = (rec or {}).get(users.TEST_ACCOUNT_KEY)
- if not isinstance(mark, dict):
- return None
- return {"copiedFrom": mark.get("copied_from"), "createdBy": mark.get("created_by"),
- "createdAt": mark.get("created_at")}
+ "perms_v": int(rec.get("perms_v") or 0)}
def _access_summary(rec, uname="", governed=None, st=None, surface_keys=None):
@@ -464,9 +454,6 @@ def update_user(username: str, body: dict = Body(default=None),
if result == "last_admin":
raise err(400, "last_admin",
"an active tenant must retain at least one administrator")
- if result in _TEST_ACCOUNT_REFUSALS:
- # R42: a test account gets no email and no admin role, refused in the core writer.
- raise err(400, result, _TEST_ACCOUNT_REFUSALS[result])
if result != "updated":
raise err(404, "no_such_user", f"no account named {uname!r}")
@@ -552,164 +539,6 @@ def expire_password(username: str, session: Session = Depends(admin_gate)):
return Response(status_code=204)
-# ── S79-T63 (R28, R42): test accounts that copy one member's permissions exactly ─────────────────
-_TEST_ACCOUNT_REFUSALS = {
- "test_account_no_email": "A test account cannot have an email address.",
- "test_account_no_whatsapp": "A test account cannot connect WhatsApp.",
- "test_account_not_admin": "A test account cannot be an administrator.",
-}
-
-
-def _copy_refusal(exc):
- return err(exc.status, exc.code, str(exc))
-
-
-#: The en and em dash, built from code points so this file carries no dash character (rule 2).
-def _copy_failure(exc, verb):
- # A store error can carry any text; rule 2 keeps dashes off the screen.
- import dash_norm
- cause = dash_norm.plain(exc.cause).strip().rstrip(".")
- message = f"The test account was not {verb}. {cause}."
- if exc.leftovers:
- message += (" These parts could not be removed: " + ", ".join(exc.leftovers)
- + ". Remove the test account again to finish.")
- elif verb == "created":
- message += " Nothing was kept."
- return err(503, "test_account_failed", message)
-
-
-def _test_account_steps(session):
- """Copy steps for grant stores outside `core`: Connector grants, through their own route."""
- import core.perm_scope as perm_scope
- import routes_keychain as keychain
-
- def held():
- doc = session.runtime.get(perm_scope.CONNECTOR_GRANT_KEY, fresh=True)
- if doc is not None and not isinstance(doc, dict):
- raise RuntimeError("connector access could not be read")
- return doc or {}
-
- def copy_grants(username, source):
- known = {kind["key"] for kind in keychain.connector_grant_kinds()}
- kinds = sorted(k for k in (held().get(source) or ()) if k in known)
- if kinds:
- keychain.put_connector_grants(username, {"granted": kinds}, session)
-
- def clear_grants(username):
- if username in held():
- keychain.put_connector_grants(username, {"granted": []}, session)
-
- return [("connector access", copy_grants, clear_grants)]
-
-
-@router.get("/admin/users/{username}/test-account/preview")
-def preview_test_account(username: str, session: Session = Depends(admin_gate)):
- """What a test account copied from this member would carry, before anything is written."""
- from core import test_accounts, user_tables
- reg = _registry()
- try:
- plan = test_accounts.plan(username, reg, tenant=_tenant_of(session.user),
- st=session.runtime)
- lent = user_tables.lend_defs(session.runtime, schema_only=True)
- except test_accounts.CopyRefused as exc:
- raise _copy_refusal(exc)
- except Exception as exc: # noqa: BLE001
- raise err(503, "store_unavailable", "Access unavailable. Try again.") from exc
- limits = _source_limits(plan["source"], reg.get(plan["source"]), session)
- body = test_accounts.preview(plan, st=lent, limits=limits)
- _label_grants(session, body["copy"]["grants"])
- return body
-
-
-def _source_limits(uname, rec, session):
- """The source's row limits, phrased for the administrator; None if they cannot be read.
-
- Field walls are lifted for the phrasing only, so a condition on a hidden field is named
- instead of reading "Limited by your access". The filters themselves are the source's.
- """
- import core.perm_scope as perm_scope
- try:
- reader = deepcopy(users._public(uname, rec))
- for entry in (reader.get("perms") or {}).values():
- if isinstance(entry, dict):
- entry.pop("hiddenFields", None)
- return perm_scope.scope_limits(reader, st=session.runtime)
- except Exception: # noqa: BLE001
- return None
-
-
-#: What a preview row says when a shared item's name cannot be read. Never a raw id.
-_GRANT_FALLBACK = {"view": "A shared view", "folder": "A shared folder", "database": "A database",
- "field": "A shared field", "cohort": "A locked view", "workbook": "A shared sheet"}
-#: Naming stops after this many seconds; every grant is still listed, the rest by kind.
-_LABEL_BUDGET_S = 3.0
-
-
-def _label_grants(session, grants):
- """Name each grant through the share door's own resolver (`routes_shares._object_ref`)."""
- import time
- try:
- from routes_shares import _object_ref
- except Exception: # noqa: BLE001
- _object_ref = None
- deadline = time.monotonic() + _LABEL_BUDGET_S
- for grant in grants:
- if grant.get("label"):
- continue
- label = None
- if time.monotonic() < deadline:
- try:
- if grant["kind"] == "workbook":
- # `_object_ref` has no workbook branch; the workbook's own name is the label.
- import core.sheets as sheets
- label = str(sheets.get_workbook(session.runtime, grant["oid"]).get("name") or "")
- elif _object_ref is not None:
- label = str(_object_ref(session, grant["kind"], grant["oid"])[0] or "")
- except Exception: # noqa: BLE001
- label = None
- label = (label or "").strip()
- grant["label"] = label or _GRANT_FALLBACK.get(grant["kind"], "A shared item")
-
-
-@router.post("/admin/test-accounts", status_code=201)
-def create_test_account(body: dict = Body(default=None), session: Session = Depends(admin_gate)):
- """Create a test account whose permissions copy `source` exactly. The password shows once."""
- from core import test_accounts
- body = body if isinstance(body, dict) else {}
- _require_store()
- try:
- made = test_accounts.create(body.get("source"), admin=session.uname,
- tenant=_tenant_of(session.user), st=session.runtime,
- steps=_test_account_steps(session))
- except test_accounts.CopyRefused as exc:
- raise _copy_refusal(exc)
- except test_accounts.CopyFailed as exc:
- raise _copy_failure(exc, "created")
- except Exception as exc: # noqa: BLE001
- raise err(503, "store_unavailable", "The test account was not created. Try again.") from exc
- rec = _registry().get(made["username"])
- if not isinstance(rec, dict):
- raise err(503, "store_unavailable", "The test account was not saved. Try again.")
- return {"user": _view(made["username"], rec), "password": made["password"]}
-
-
-@router.delete("/admin/test-accounts/{username}", status_code=204)
-def delete_test_account(username: str, session: Session = Depends(admin_gate)):
- """Remove a test account, its sign in and every grant naming it, in one call."""
- from core import test_accounts
- _require_store()
- try:
- test_accounts.remove(username, tenant=_tenant_of(session.user), st=session.runtime,
- granter=session.uname, steps=_test_account_steps(session))
- except test_accounts.CopyRefused as exc:
- raise _copy_refusal(exc)
- except test_accounts.CopyFailed as exc:
- raise _copy_failure(exc, "fully removed")
- except Exception as exc: # noqa: BLE001
- raise err(503, "store_unavailable", "The test account was not removed. Try again.") from exc
- return Response(status_code=204)
-
-
# ── C-PERM (wave 15): per-module access + permanent filters + hidden fields ──────────────────
#: The modules the permission editor governs — the GRID surfaces, the ones with a field schema
#: to filter and hide. Amendment 6 shipped the wall on `customer_data` alone and said this list
diff --git a/api/routes_amazon_projects.py b/api/routes_amazon_projects.py
index 1c2a8a46f2416b4a4e3d3c760467953cf111a33f..83d1fa7a6ece23be66c8a99585c3c9881abbd4fa 100644
--- a/api/routes_amazon_projects.py
+++ b/api/routes_amazon_projects.py
@@ -12,7 +12,7 @@ from pathlib import Path
import sys
from typing import Any
-from fastapi import APIRouter, Body, Depends, HTTPException
+from fastapi import APIRouter, Body, Depends, HTTPException, Query
from core import amazon_projects as projects
from core import amazon_project_results as project_results
@@ -219,7 +219,7 @@ def _distinct(found, pairs):
def cell_values(records):
- """R19 and R67: the average progress as a stored pct and "N of M completed", blank with no counted project."""
+ """Store average progress with Completed at 100 and the completion count, blank when none count."""
counted = projects.summary(records)
if not counted["total"]:
return {PROGRESS_KEY: None, LABEL_KEY: None}
@@ -343,28 +343,34 @@ def _result_sources(session, targets):
return available
+class ProjectMirror(sku_action_results.Mirror):
+ """Read a deduplicated project SKU union and retain source evidence for every target day."""
+
+ def __init__(self, con, skus, available):
+ super().__init__(con, {}, available)
+ self.skus, self.coverage = tuple(skus), []
+
+ def entity(self):
+ self.field(sku_action_results.ECONOMICS, "msku")
+ return '"msku" IN (' + ", ".join("?" for _ in self.skus) + ")", self.skus
+
+ def raw(self, table, days, values, **kwargs):
+ plan = super().raw(table, days, values, **kwargs)
+ sku = self.field(table, *kwargs["sku_fields"])
+ head, tail = plan.sql.split(" FROM ", 1)
+ coverage = sku_action_results.Plan(
+ head + ", " + sku_action_results._q(sku) + ' AS "__project_member" FROM ' + tail,
+ plan.args, plan.sources)
+ self.coverage.append((table, tuple(days), coverage))
+ return plan
+
+
def _result_figure(con, config, bounds, skus, available):
"""Reuse daily metric definitions and reconcile their union against separate source totals."""
if not skus:
return {"state": "unavailable", "reason": "Assign an Amazon target before measuring results."}
- class ProjectMirror(sku_action_results.Mirror):
- def entity(self):
- self.field(sku_action_results.ECONOMICS, "msku")
- return '"msku" IN (' + ", ".join("?" for _ in skus) + ")", tuple(skus)
-
- def raw(self, table, days, values, **kwargs):
- plan = super().raw(table, days, values, **kwargs)
- sku = self.field(table, *kwargs["sku_fields"])
- head, tail = plan.sql.split(" FROM ", 1)
- coverage = sku_action_results.Plan(
- head + ", " + sku_action_results._q(sku) + ' AS "__project_member" FROM ' + tail,
- plan.args, plan.sources)
- self.coverage.append((table, tuple(days), coverage))
- return plan
-
- mirror = ProjectMirror(con, {}, available)
- mirror.coverage = []
+ mirror = ProjectMirror(con, skus, available)
try:
mirror.select_scope(bounds)
except sku_action_results.SourceUnavailable as exc:
@@ -406,6 +412,81 @@ def _complete_result_membership(con, coverage, bounds, sources, skus, names=("cu
return True
+_SERIES_UNAVAILABLE = {
+ "margin": "Margin requires daily net profit with verified Odoo or typed COGS. "
+ "amzn_economics alone cannot reconcile that cost.",
+ "returns": "Returns require FBA and MFN customer return events. "
+ "amzn_economics has no independent returned-unit aggregate.",
+ "returns_pct": "Returns % requires returned units over shipped units. "
+ "amzn_economics cannot independently reconcile those quantities.",
+ "refunds_pct": "Refunds % requires deduplicated refund units over shipped units. "
+ "amzn_economics cannot independently reconcile those quantities.",
+ "roas": "ROAS requires attributed ad sales over ad spend. "
+ "amzn_economics has no independent attributed ad-sales aggregate.",
+}
+
+
+def _series_figure(key, figure, bounds):
+ """Adapt existing sales and units definitions without publishing partial source windows."""
+ if key in _SERIES_UNAVAILABLE:
+ return {"reason": _SERIES_UNAVAILABLE[key]}
+ if figure.get("reason"):
+ return {"reason": figure["reason"]}
+ current = figure.get("current")
+ if not current or current["status"] != "available":
+ return {"reason": (current or {}).get("reason") or "Some target source days are unavailable."}
+ if key != "sales_yoy":
+ return {"total": current["value"], "points": current["series"], "unit": figure["unit"],
+ "reason": "Today is incomplete and excluded." if bounds["to"] == bounds["current"][1].isoformat() else None}
+ prior = figure["comparison"]
+ if prior["status"] != "available" or prior["value"] is None or prior["value"] <= 0:
+ return {"reason": prior.get("reason") or "Sales YoY requires positive sales in the matching prior window."}
+ points = []
+ for now, before in zip(current["series"], prior["series"]):
+ if before["value"] is None or before["value"] <= 0:
+ return {"reason": "Sales YoY requires positive sales on every matching prior day."}
+ points.append({"date": now["date"], "value": 100 * (now["value"] - before["value"]) / before["value"]})
+ return {"total": 100 * (current["value"] - prior["value"]) / prior["value"], "points": points,
+ "reason": "Today is incomplete and excluded." if bounds["to"] == bounds["current"][1].isoformat() else None}
+
+
+def _series_economics_total(con, key, bounds, skus, available):
+ """Aggregate original economics rows without the display plan, daily rollup or display total."""
+ if key in _SERIES_UNAVAILABLE:
+ return {"value": None, "reason": _SERIES_UNAVAILABLE[key]}
+ if not skus or not bounds["days"]:
+ return {"value": None, "reason": "No completed target days are available in this range."}
+ mirror = ProjectMirror(con, skus, available)
+ try:
+ mirror.select_scope(bounds)
+ field = mirror.field(sku_action_results.ECONOMICS,
+ "sales_net_units_sold" if key == "units" else "sales_net_product_sales_amount")
+ number = f'TRY_CAST({sku_action_results._q(field)} AS DOUBLE)'
+ day = sku_action_results._day("start_date")
+ marks = ", ".join("?" for _ in skus)
+
+ def total(days):
+ row = con.execute(
+ f'SELECT SUM({number}), count(*), count({number}), count(DISTINCT (msku, {day})) '
+ f'FROM amzn_economics WHERE msku IN ({marks}) AND marketplace_id = ? '
+ f'AND sales_net_product_sales_currency_code = ? AND {day} >= ? AND {day} < ?',
+ (*skus, *mirror.scope, *days)).fetchone()
+ if (not row[1] or row[1] != row[2] or row[3] != len(skus) * bounds["days"]
+ or not project_results._finite(row[0])):
+ raise sku_action_results.SourceUnavailable("amzn_economics: some target values or days are unavailable.")
+ return float(row[0])
+
+ value = total(bounds["current"])
+ if key == "sales_yoy":
+ prior = total(bounds["comparison"])
+ if prior <= 0:
+ raise sku_action_results.SourceUnavailable("Sales YoY requires positive prior sales.")
+ value = 100 * (value - prior) / prior
+ return {"value": value, "reason": None}
+ except sku_action_results.SourceUnavailable as exc:
+ return {"value": None, "reason": str(exc)}
+
+
@router.get("")
def row_projects(grain: str | None = None, key: str | None = None,
session: Session = Depends(require_session)):
@@ -508,6 +589,61 @@ def project_result_figures(project_id: str, session: Session = Depends(require_s
con.close()
+@router.get("/{project_id}/series")
+def project_result_series(project_id: str, start: str | None = Query(None, alias="from"),
+ end: str | None = Query(None, alias="to"), scope: str = Query("all"),
+ session: Session = Depends(require_session)):
+ """Serve the preset daily charts within the intersection of project targets and the modal filter."""
+ runtime = _reader(session)
+ record = _call(projects.get, runtime, project_id)
+ targets = _call(projects.targets, runtime, project_id)
+ con = None
+ try:
+ bounds = project_results.series_window(record, start, end, today=projects._today())
+ selected = project_results.series_scope(scope)
+ selected_targets = ([{"grain": "parent_asin" if selected["grain"] == "parent" else selected["grain"],
+ "key": key} for key in selected["keys"]] if selected else [])
+ available = _result_sources(session, targets + selected_targets)
+ con = routes_sku_actions._results_mirror(runtime)
+ reason = None
+ try:
+ skus = projects.resolve_target_skus(con, targets)
+ if selected:
+ selected_skus = set(projects.resolve_target_skus(con, selected_targets))
+ skus = [sku for sku in skus if sku in selected_skus]
+ if not skus:
+ reason = "No project targets match the selected scope."
+ elif not bounds["days"]:
+ reason = "Today is incomplete. No completed days are available in this range."
+ except projects.MirrorUnreadable as exc:
+ skus, reason = [], str(exc)
+ figures = {}
+
+ def read(key):
+ if reason:
+ return {"reason": reason}
+ metric = "sales" if key == "sales_yoy" else key
+ if metric not in _SERIES_UNAVAILABLE and metric not in figures:
+ figures[metric] = _result_figure(con, {"metric": metric}, bounds, skus, available)
+ return _series_figure(key, figures.get(metric, {}), bounds)
+
+ return project_results.series(record, bounds=bounds, read=read,
+ independent=lambda key: _series_economics_total(con, key, bounds, skus, available))
+ except HTTPException:
+ raise
+ except (project_results.ValidationFailed, sku_action_results.ValidationFailed):
+ raise err(409, "project_results_not_validated",
+ "A series total does not tie to its independent Amazon economics aggregate.") from None
+ except ValueError as exc:
+ raise err(400, "invalid_project_results", str(exc)) from None
+ except Exception as exc: # noqa: BLE001
+ print(f"[amazon-projects] series failed: {type(exc).__name__}: {exc}", file=sys.stderr)
+ raise err(503, "project_results_unavailable", "Project result sources are unavailable.") from None
+ finally:
+ if con is not None:
+ con.close()
+
+
@router.patch("/{project_id}")
def edit_project(project_id: str, body: Any = Body(default=None), session: Session = Depends(require_session)):
"""Change the named fields. Targets named here are added."""
@@ -522,8 +658,11 @@ def edit_project(project_id: str, body: Any = Body(default=None), session: Sessi
record = _call(projects.update, runtime, project_id, values, by=_by(session))
if targets:
record = _call(projects.add_targets, runtime, project_id, targets, by=_by(session))["project"]
- # Only status and progress move a cell: a name or note edit repaints only the targets it adds.
- _repaint(runtime, _target_pairs(runtime, project_id) if {"status", "progress"} & set(values) else targets)
+ # A status or progress edit repaints every assigned target, including when a project reopens.
+ if {"status", "progress"} & set(values):
+ _repaint(runtime, _target_pairs(runtime, project_id))
+ elif targets:
+ _repaint(runtime, targets)
return {"project": _project(record, projects._today())}
diff --git a/api/routes_connectors.py b/api/routes_connectors.py
index c739077679dd4701c6bf3914c8bea6e90c19c7f5..90934cc21b0a170373992543bd73c814386a9d2f 100644
--- a/api/routes_connectors.py
+++ b/api/routes_connectors.py
@@ -65,7 +65,7 @@ from core import schema_overlay
#: one question, and this module used to answer it with an unguarded `os.environ.get("ODOO_URL")`
#: while `routes_keychain` answered it correctly. Importing the correct one is the fix; writing a
#: second correct one here would leave the same two-copies shape that produced the defect.
-from routes_keychain import TENANT_WIDE_TYPES, allowed_scopes, env_odoo_available
+from routes_keychain import TENANT_WIDE_TYPES, allowed_scopes, env_odoo_available, stored_business_tz
#: ⛔⛔ W49-T16 / C3 — THE DEPENDENTS ROUTER IS MOUNTED FROM HERE, NOT FROM `main.py`. In a
#: worktree-parallel wave `main.py` lives in exactly ONE lane's fence, so a router built in any
#: other lane has nowhere to be mounted from (five consecutive waves lost an `include_router`
@@ -1095,15 +1095,25 @@ def _contact_rows(rt, uname, is_admin, incomplete):
def read(source, keys, fn):
try:
fn()
- except Exception: # noqa: BLE001
+ except Exception as exc: # noqa: BLE001
+ logging.getLogger(__name__).warning("contact_check=%s exception=%s",
+ source, type(exc).__name__)
for key in keys:
by_key[key]["state"] = "unavailable"
- incomplete.append({"source": source, "cause": "Connection status unavailable.",
+ label = {"whatsapp": "WhatsApp", "slack": "Slack", "mcp": "MCP",
+ "loopable-api": "Loopable API"}[source]
+ incomplete.append({"source": source, "cause": f"{label} connection status unavailable.",
"recommendation": "Try again."})
def whatsapp():
- import whatsapp_identity
- if whatsapp_identity.status(rt.key, uname).get("state") == "connected":
+ from core import account_rows, users
+ from whatsapp_identity import WhatsAppRefusal
+ # Directory status reads the reader's canonical account, never registration challenges.
+ record = account_rows.get(uname, st=users.store, fresh=True)
+ if not isinstance(record, dict) or str(record.get("tenant") or "royal-imports") != rt.key:
+ raise WhatsAppRefusal("This account is not available in this workspace.")
+ connection = record.get("whatsapp")
+ if isinstance(connection, dict) and connection.get("phone"):
by_key["whatsapp"]["state"] = "connected"
def slack():
@@ -3056,6 +3066,88 @@ def _amazon_mirror_completion(rt):
return None, False
+def _amazon_table_schedule(rt, connection_id=None, now=None):
+ """Read account cadence metadata without restoring a mirror or reading source rows."""
+ import duckdb
+ from core import keychain
+ from harness import amazon_store as az, datastore
+
+ now = now or datetime.now(timezone.utc)
+ result = {"tableSchedule": [], "scheduleState": "unavailable",
+ "displayTimezone": stored_business_tz(rt) or "UTC"}
+
+ def read(cursor, entries):
+ schedule = []
+ for entry in entries:
+ cid = entry["connection_id"]
+ prefix = az._account_prefix(cid)
+ keys = [prefix + az.entity_key(family + table)
+ for table in az.TABLES for family in ("", "sync_tick_", az.WATERMARK_PREFIX)]
+ rows = cursor.execute(
+ "SELECT entity, phase, rows, updated_at, "
+ "CASE WHEN phase = 'throttled' THEN cursor_wd ELSE NULL END FROM _sync_state "
+ "WHERE entity IN (SELECT unnest(?))", [keys]).fetchall()
+ markers = {entity: {"state": phase, "rows": count, "updated_at": _sync_stamp(at),
+ "detail": _sync_stamp(detail)}
+ for entity, phase, count, at, detail in rows}
+ states = {table: state for table in az.TABLES
+ if (state := markers.get(prefix + az.entity_key("sync_tick_" + table)))}
+ if any(not state["updated_at"] for state in states.values()):
+ raise ValueError("Amazon cadence metadata has an invalid timestamp")
+ # C1 owns every due time, including throttle retries and calendar periods.
+ for row in az.sync_schedule(states, now):
+ table = row["table"]
+ tick = states.get(table) or {}
+ watermark = markers.get(prefix + az.entity_key(az.WATERMARK_PREFIX + table)) or {}
+ landed = markers.get(prefix + az.entity_key(table)) or {}
+ completed = (tick if tick.get("state") == "done"
+ and landed.get("state") not in ("refused", "failed", "throttled") else watermark)
+ schedule.append({**row, "connectionId": cid, "account": entry["label"],
+ "lastSyncAt": (completed.get("updated_at")
+ if completed.get("state") == "done" else None),
+ "state": (landed.get("state")
+ if landed.get("state") in ("refused", "failed", "throttled")
+ else tick.get("state") or "never_run")})
+ return schedule
+
+ try:
+ path = az._tenant_mirror_path(rt.key, rt=rt)
+ if not keychain.list_entries(rt, "amazon_sp"):
+ return {**result, "scheduleState": "empty"}
+ entries = az.account_entries(rt.key, rt)
+ cid = str(connection_id or "").strip()
+ if cid and cid != "default":
+ entries = [entry for entry in entries if entry["connection_id"] == cid]
+ if not entries:
+ return result
+ if not path.is_file():
+ return result
+ if path.resolve() == Path(datastore.DB_PATH).resolve():
+ schedule = read(datastore.ro_cursor(), entries)
+ else:
+ with datastore.bounded_connection(duckdb.connect(str(path), read_only=True)) as cursor:
+ schedule = read(cursor, entries)
+ return {**result, "tableSchedule": schedule, "scheduleState": "ready"}
+ except Exception as exc:
+ raise_if_resource_error(exc)
+ return result
+
+
+def _amazon_sync_staleness(completed, schedule, now=None):
+ """Use the fastest Amazon table's cadence and Odoo's existing grace period."""
+ import odoo_relational as rel
+ fastest = max(amazon_client.CADENCE[table]["perDay"]
+ for table in amazon_client.GROUP_A)
+ allowed = 86400 / fastest + rel.STALE_GRACE_SECONDS
+ stamps = [row.get("lastSyncAt") for row in schedule if row["perDay"] == fastest]
+ if not stamps:
+ stamps = [completed]
+ now = now or datetime.now(timezone.utc)
+ stale = any((now - datetime.fromisoformat(stamp)).total_seconds() > allowed
+ for value in stamps if (stamp := _sync_stamp(value)))
+ return {"stale": stale, "staleAfterSeconds": allowed}
+
+
def _daily_completed_sync(rt, connector_key):
"""Read the stored last good sync; never create a mirror or contact a provider."""
if connector_key == "amazon_sp":
@@ -3154,8 +3246,11 @@ def connector_sync_status(connector_key: str, connection_id: str = None,
if connector_key not in registered:
raise err(404, "sync_provider_unknown",
f"{connector_key} is not registered for connector sync")
+ from core import keychain
+ import odoo_relational as rel
rt = (_odoo_status_reads(session.runtime) if connector_key == "odoo"
- else _ConnectorStatusReads(session.runtime, (connector_sync.JOBS_KEY, connector_sync.STATE_KEY, connector_sync.STATUS_KEY)
+ else _ConnectorStatusReads(session.runtime, (connector_sync.JOBS_KEY, connector_sync.STATE_KEY,
+ connector_sync.STATUS_KEY, keychain.KEY, rel.CONFIG_KEY)
if connector_key == "amazon_sp" else ()))
resolved_connection = (_odoo_connection_slot(rt, connection_id)
if connector_key == "odoo" else connection_id)
@@ -3179,9 +3274,15 @@ def connector_sync_status(connector_key: str, connection_id: str = None,
if connector_key == "amazon_sp" and completed is None and known and _amazon_restore_pending(rt):
completed = _amazon_saved_completion(rt)
answer["restoring"] = True
- answer.update(schedule="daily", lastCompletedAt=completed, lastSyncAt=completed,
- lastCompletedKnown=known, canSync=bool(session.admin),
- **_daily_sync_staleness(completed))
+ answer.update(lastCompletedAt=completed, lastSyncAt=completed,
+ lastCompletedKnown=known, canSync=bool(session.admin))
+ if connector_key == "amazon_sp":
+ now = datetime.now(timezone.utc)
+ schedule = _amazon_table_schedule(rt, resolved_connection, now)
+ answer.update(schedule="per_table", **schedule,
+ **_amazon_sync_staleness(completed, schedule["tableSchedule"], now))
+ else:
+ answer.update(schedule="daily", **_daily_sync_staleness(completed))
return _safe_connector_status(connector_key, answer)
diff --git a/api/routes_external.py b/api/routes_external.py
index 65dd1d58b022b28ec0fa83e3216a61609e4260a8..79f0dc7a7ce6f0f219e93f3746aea28224471cf6 100644
--- a/api/routes_external.py
+++ b/api/routes_external.py
@@ -153,6 +153,20 @@ def _activity_store_view(real, insert):
return view
+def _sheet_row_revision(tx, key):
+ """Read the header's store revision on the same transaction as its sheet rows."""
+ from core import store_pg, view_audit
+ if isinstance(tx, sheets._PgSheetRows):
+ row = tx.con.execute(store_pg._q(
+ "SELECT rev, updated_at FROM {tbl} WHERE key = %s", "store_kv", tx.slug),
+ (tx.namespace + key,)).fetchone()
+ return store_pg._revision_value(row[0], row[1]) if row else store_pg._revision_value(None, None)
+ reader = getattr(type(tx), "revision", None)
+ if callable(reader):
+ return reader(tx, key)
+ raise view_audit.RevisionUnverifiable("The Sheet transaction cannot verify its header revision.")
+
+
class _PreviewRuntime:
"""Lazy copy-on-write view of one tenant runtime; no method may reach a real writer.
@@ -180,11 +194,20 @@ class _PreviewRuntime:
if key not in self._docs:
import application_registry
value = application_registry._read_strict(self._real, key)
+ revision = None
+ if key.startswith(sheets.WORKBOOK_KEY_PREFIX) and sheets.is_row_workbook(value):
+ # Hydrate through the reader, never delegate a native writer to this preview.
+ with sheets._sheet_rows_transaction(self._real) as tx:
+ if tx is None:
+ raise err(503, "workbook_unavailable", "The workbook snapshot is unavailable. Try again.")
+ head = tx.get(key)
+ value = sheets._verify_row_layout(tx, head) if sheets.is_row_workbook(head) else head
+ revision = _sheet_row_revision(tx, key)
value = copy.deepcopy(value)
self._initial[key] = copy.deepcopy(value)
self._docs[key] = value
try:
- revision = self._real.revision(key)
+ revision = revision if revision is not None else self._real.revision(key)
except Exception:
revision = None
self._base_revisions[key] = copy.deepcopy(revision)
@@ -328,6 +351,70 @@ class _ConfirmationRuntime:
def tenant(self):
return getattr(self._real, "tenant", None)
+ @contextmanager
+ def sheet_rows_transaction(self, *, write=False):
+ """Guard native Sheet writes under the header lock with the preview's store revision."""
+ from core import view_audit
+ with sheets._sheet_rows_transaction(self._real, write=write) as tx:
+ if tx is None:
+ yield tx
+ return
+ owner = self
+ held = {}
+
+ class GuardedRows:
+ def get(self, key, *, lock=False, **selection):
+ if lock and not write:
+ raise view_audit.RevisionUnverifiable("A Sheet read cannot acquire write authority.")
+ value = tx.get(key, lock=lock, **selection)
+ if lock:
+ expected = owner._expected.get(key)
+ rev = expected.get("rev") if isinstance(expected, dict) else None
+ if (not key.startswith(sheets.WORKBOOK_KEY_PREFIX) or type(rev) is not int
+ or held):
+ raise view_audit.RevisionUnverifiable("The Sheet writer addressed an unpreviewed header.")
+ current = _sheet_row_revision(tx, key)
+ if current.get("rev") != rev or ("token" in expected and current.get("token") != expected["token"]):
+ raise view_audit.ViewRevisionStale(rev_current=current.get("rev"), rev_sent=rev)
+ before = (sheets._verify_row_layout(tx, value) if sheets.is_row_workbook(value)
+ else tx.get(key))
+ held.update(key=key, before=copy.deepcopy(before), head=value)
+ return value
+
+ def revision(self, key):
+ return _sheet_row_revision(tx, key)
+
+ def put(self, key, value):
+ header = held.get("key")
+ if not header or (key != header and key != sheets.sheet_row_key(
+ header[len(sheets.WORKBOOK_KEY_PREFIX):], value.get("id", ""))):
+ raise view_audit.RevisionUnverifiable("The Sheet writer addressed an unpreviewed workbook.")
+ if key == header and value and value.get("id") != header[len(sheets.WORKBOOK_KEY_PREFIX):]:
+ raise view_audit.RevisionUnverifiable("The Sheet header identity changed.")
+ tx.put(key, value)
+
+ def delete(self, key):
+ head = held.get("head") or {}
+ if key not in {sheets.sheet_row_key(head["id"], ref["id"])
+ for ref in head.get("rowSheets", [])}:
+ raise view_audit.RevisionUnverifiable("The Sheet writer addressed an unpreviewed sheet.")
+ tx.delete(key)
+
+ yield GuardedRows()
+ if held:
+ key = held["key"]
+ head = tx.get(key)
+ after = sheets._verify_row_layout(tx, head) if sheets.is_row_workbook(head) else head
+ produced = _sheet_row_revision(tx, key)
+ if held:
+ self._expected[key] = copy.deepcopy(produced)
+ if held["before"] != after:
+ previous = self._committed_documents.get(key)
+ self._committed_documents[key] = {
+ "before": previous["before"] if previous else held["before"],
+ "after": copy.deepcopy(after),
+ }
+
def update(self, name, callback, flush="sync", expect_rev=None, audit_row=None):
from core import view_audit
key = str(name)
@@ -1903,6 +1990,22 @@ def list_keys(session: Session = Depends(require_session)):
return {"keys": keys, "can_mint": _can_use_analyst(session)}
+@router.post("/keys/verify")
+def verify_owned_key(response: Response, body=Body(default=None), session: Session = Depends(require_session)):
+ """Return a private diagnosis for this session owner's key without accepting bearer sign-in."""
+ require_external_analyst(session)
+ token = body.get("token") if isinstance(body, dict) else None
+ if not isinstance(token, str) or len(token) > 1024:
+ token = ""
+ try:
+ result = external_access.verify_for_owner(
+ session.runtime, session.uname, session.tenant, token)
+ except external_access.ExternalAccessError as exc:
+ _management_refusal(exc)
+ response.headers["Cache-Control"] = "no-store"
+ return result
+
+
@router.patch("/keys/{key_id}")
def edit_key(key_id: str, body=Body(default=None), session: Session = Depends(require_session)):
"""Edit metadata and narrowing scope without ever reading or replacing the secret."""
@@ -3224,11 +3327,8 @@ def _validate_personal_view_draft(session: Session, database: str, raw: dict) ->
if any(key not in visible for key in _view_referenced_fields(config)):
raise err(400, "assistant_unknown_field",
"Every requested field must be readable in this database.")
- # The ordinary Assistant validator owns filter syntax and operands. fields=[] keeps values
- # out of this validation result while still applying the live database/field/row walls.
- filters = config.get("filters")
- _external_read_scope(
- session, database, fields=[], filters=None if filters == [] else filters)
+ # Saved predicates use Table's cleaner. The reader still validates permanent permission filters.
+ _external_read_scope(session, database, fields=[], filters=None)
def _saved_view_filters(config):
@@ -3237,9 +3337,9 @@ def _saved_view_filters(config):
_assert_query_filter_bounds([value] if isinstance(value, dict) else value)
import aios_grid
- def refuse():
+ def refuse(message=None):
raise err(400, "view_filters_unsupported",
- "These filters cannot be saved without changing their meaning. "
+ message or "These filters cannot be saved without changing their meaning. "
"Use colId, op and value leaves, with conj and children for nested groups.")
def node(raw):
@@ -3251,8 +3351,17 @@ def _saved_view_filters(config):
or not isinstance(raw[key], list) or not raw[key]):
refuse()
return {"conj": raw.get("conj", "and"), "children": [node(child) for child in raw[key]]}
- if (set(raw) - {"colId", "op", "value", "value2", "values", "id", "locked"}
- or not isinstance(raw.get("colId"), str) or raw.get("op") not in aios_grid.FILTER_OPS):
+ if any(key in raw for key in ("valueRef", "value2Ref")):
+ refuse("Cell references cannot be saved through this tool because workbook and cell access "
+ "cannot be verified here. Use literal values or a readable rhs field instead.")
+ op = raw.get("op")
+ if not isinstance(op, str) or op not in aios_grid.FILTER_OPS:
+ name = json.dumps(op[:120]) if isinstance(op, str) else "(missing or invalid)"
+ refuse(f"Unsupported filter operator {name}. Use a supported operator: "
+ + ", ".join(sorted(aios_grid.FILTER_OPS)) + ".")
+ if (set(raw) - {"colId", "op", "value", "value2", "values", "id", "locked",
+ "dateWindow", "dateMode", "rhs"}
+ or not isinstance(raw.get("colId"), str)):
refuse()
for key in ("value", "value2"):
value = raw.get(key)
diff --git a/api/routes_odoo_tables.py b/api/routes_odoo_tables.py
index e5a51eb38389a12b737c33b9aa80e6ca8e331acd..6d924e80add7612c81fd24524a61ccf17dcbee8b 100644
--- a/api/routes_odoo_tables.py
+++ b/api/routes_odoo_tables.py
@@ -1393,6 +1393,22 @@ def require_amazon_mirror(table_key, session, definition=None, st=None):
cur.close()
+def require_dynamic_odoo_mirror(table_key, session, st=None):
+ """Admit a saved dynamic definition before resolving its publication or refusal."""
+ if not str(table_key).startswith('ut_odoo_model_'):
+ return None
+ from routes_tables import _defn_or_refuse
+ try:
+ _defn_or_refuse(session, table_key, st=st, defs_only=True, scope_applied=True)
+ except HTTPException as exc:
+ if exc.status_code == 404:
+ raise err(404, 'odoo_definition_not_published',
+ 'This Odoo database has no published definition. Enable it in connection '
+ 'settings and complete its first sync before reading rows.') from exc
+ raise
+ return not _materialised_table(table_key, st=st if st is not None else session.runtime)
+
+
register_source_provider(_odoo_sources)
register_source_provider(_amazon_sources)
register_source_provider(_meta_sources)
@@ -1609,6 +1625,8 @@ def _amazon_page_source(cur, key, definition=None, *, binding=False, allow_looku
"""Resolve one admitted Amazon target and keep query failures distinct from stored tables."""
import amazon_relational as amazon
+ if (definition or {}).get('oversize'):
+ definition = {**definition, 'readThrough': True}
try:
if binding:
columns = amazon.mirror_columns(cur, amazon.SOURCE[key], strict=True)
@@ -1621,7 +1639,7 @@ def _amazon_page_source(cur, key, definition=None, *, binding=False, allow_looku
raise err(503, "amazon_source_unavailable",
"This Amazon database could not be resolved. Retry the request.") from exc
if spec is not None and spec.get("missing"):
- raise err(503, "amazon_mirror_not_loaded", spec["missing"])
+ raise err(409, "amazon_mirror_not_loaded", spec["missing"])
if spec is not None and '_connection_id' in {
name for name, _kind in amazon.mirror_columns(cur, amazon.SOURCE[key], strict=True)}:
spec = {**spec, 'cols': {**spec['cols'], '_connection_id': (
@@ -1657,7 +1675,10 @@ def _source_for(cur, table_key, *, allow_lookups=True):
if module_cache_source(key) is not None:
return None
if key.startswith("ut_odoo_model_"):
- return _rel().dynamic_model_source(cur, key)
+ try:
+ return _rel().dynamic_model_source(cur, key)
+ except _rel().Refused as exc:
+ raise err(409, 'odoo_mirror_binding_incomplete', str(exc)) from exc
if key.startswith('ut_meta_'):
return _meta_source(cur, key)
if key.startswith("ut_amzn_"):
@@ -1673,7 +1694,7 @@ def _source_for(cur, table_key, *, allow_lookups=True):
if not spec:
return None
if spec.get("missing"):
- raise err(503, "amazon_mirror_not_loaded", spec["missing"])
+ raise err(409, "amazon_mirror_not_loaded", spec["missing"])
from harness import datastore
if _rel().base_table_key(table_key) == _rel().BILLS_KEY:
try:
@@ -1712,7 +1733,7 @@ def _source_for(cur, table_key, *, allow_lookups=True):
if not cols:
if key.startswith("ut_amzn_"):
import amazon_relational
- raise err(503, "amazon_mirror_not_loaded",
+ raise err(409, "amazon_mirror_not_loaded",
amazon_relational.missing_source(key)["missing"])
return None # the mirror has no such table on this deployment
have_by_alias[alias] = cols
@@ -2642,16 +2663,33 @@ def _materialised_table(table_key, *, cur=None, st=None):
if module_cache_source(table_key) is not None:
return True
if str(table_key).startswith("ut_odoo_model_"):
- if st is not None and (_ut().get_definition(table_key, st=st) or {}).get("readThrough") is not True:
+ definition = _ut().get_definition(table_key, st=st) if st is not None else {}
+ mandatory = bool((definition or {}).get('readThrough') or (definition or {}).get('oversize'))
+ if st is not None and not mandatory:
return True
own_cursor = cur is None
if own_cursor:
if st is None:
return True
- cur = _table_cursor(st, table_key)
+ try:
+ cur = _table_cursor(st, table_key)
+ except _rt().DatastoreMismatch as exc:
+ raise err(409, 'cross_tenant_store', str(exc)) from exc
+ except _rel().Refused as exc:
+ raise err(409, 'odoo_mirror_binding_incomplete', str(exc)) from exc
+ except RuntimeError as exc:
+ raise err(409, 'odoo_mirror_not_loaded', str(exc)) from exc
try:
- return _rel().dynamic_model_source(cur, table_key,
- tenant_key=getattr(st, "key", None)) is None
+ try:
+ spec = _rel().dynamic_model_source(cur, table_key,
+ tenant_key=getattr(st, "key", None))
+ except _rel().Refused as exc:
+ raise err(409, 'odoo_mirror_binding_incomplete', str(exc)) from exc
+ if mandatory and spec is None:
+ raise err(409, 'odoo_mirror_not_loaded',
+ 'This Odoo database has no complete mirror publication. Complete its '
+ 'sync in connection settings before reading rows.')
+ return spec is None
finally:
if own_cursor and cur is not None:
cur.close()
@@ -2663,7 +2701,7 @@ def _materialised_table(table_key, *, cur=None, st=None):
own_cursor = cur is None
cur = datastore.ro_cursor() if own_cursor else cur
try:
- definition = _ut().get(table_key, st=st) if st is not None else None
+ definition = _ut().get_definition(table_key, st=st) if st is not None else None
spec = _amazon_page_source(cur, table_key, definition, allow_lookups=False)
return spec is None
finally:
@@ -4009,6 +4047,8 @@ def resolved_fields(table_key, defn):
if str(table_key).startswith("ut_amzn_"):
import amazon_relational as amazon
connected = table_key in amazon.SOURCE
+ elif str(base).startswith('ut_odoo_model_'):
+ connected = True
else:
connected = table_key in _sources()
if not connected:
@@ -4108,7 +4148,7 @@ def _compiled_read(table_key, session, *, filters=None, filterConj="and", sorts=
# try, not inside it.
try:
cur, mismatch = (_mirror_cur(session, table_key)
- if (table_key.startswith(("ut_amzn_", "ut_meta_"))
+ if (table_key.startswith(("ut_amzn_", "ut_meta_", "ut_odoo_model_"))
or rel.base_table_key(table_key) == rel.SHIPMENTS_KEY
or rel.base_table_key(table_key) in rel.stock_source_specs())
else _mirror_cur(session))
@@ -5117,8 +5157,10 @@ def odoo_table_rows(table_key: str,
ids = _record_ids(body)
# Admission resolves this Amazon database without publishing a process-wide binding.
- page = _page_defs(session) if str(table_key).startswith("ut_amzn_") else None
+ page = _page_defs(session) if str(table_key).startswith(("ut_amzn_", "ut_odoo_model_")) else None
admitted = require_amazon_mirror(table_key, session, st=page[0] if page else None)
+ if str(table_key).startswith('ut_odoo_model_'):
+ admitted = True if require_dynamic_odoo_mirror(table_key, session, st=page[0]) else None
if (admitted is None if page is not None else _materialised_table(table_key, st=session.runtime)):
return _audit_personal_response(table_key, _materialised_window(
table_key, session, offset, limit, filters, filterConj,
@@ -5746,8 +5788,10 @@ def odoo_table_aggregates(table_key: str, body: dict = Body(default=None),
if group_limit is not None and (type(group_limit) is not int or not 1 <= group_limit <= MAX_GROUPS):
raise err(400, "bad_selection", f"groupLimit must be a page size from 1 to {MAX_GROUPS}")
- page = _page_defs(session) if str(table_key).startswith("ut_amzn_") else None
+ page = _page_defs(session) if str(table_key).startswith(("ut_amzn_", "ut_odoo_model_")) else None
admitted = require_amazon_mirror(table_key, session, st=page[0] if page else None)
+ if str(table_key).startswith('ut_odoo_model_'):
+ admitted = True if require_dynamic_odoo_mirror(table_key, session, st=page[0]) else None
if (admitted is None if page is not None else _materialised_table(table_key, st=session.runtime)):
with _materialised_read(table_key, session, query, payload.get("related"), ids=ids) as read:
diff --git a/api/sync_worker.py b/api/sync_worker.py
index 76112aa19527e381ab387f23a1ac3862bf1def6b..184b55e7cb9d7af42ef9575c3ceddd593152fe38 100644
--- a/api/sync_worker.py
+++ b/api/sync_worker.py
@@ -35,9 +35,10 @@ def configure(env=None):
worker_db = worker_dir / f"{safe}.duckdb"
env["AIOS_DUCKDB_STORE_DIR"] = str(worker_dir)
env["AIOS_DUCKDB_PATH"] = str(worker_db)
- # The launcher (request_class.py --env) sizes these to the box. Unset: the 2 vCPU values.
- env["AIOS_DUCKDB_MEMORY"] = env.get("AIOS_WORKER_DUCKDB_MEMORY") or "4GB"
+ # Leave room for Python and the web processes when a full Brand Analytics week is parsed.
+ env["AIOS_DUCKDB_MEMORY"] = env.get("AIOS_WORKER_DUCKDB_MEMORY") or "6GB"
env["AIOS_DUCKDB_THREADS"] = env.get("AIOS_WORKER_DUCKDB_THREADS") or "1"
+ env["AIOS_DUCKDB_MAX_TEMP_SIZE"] = env.get("AIOS_WORKER_DUCKDB_MAX_TEMP_SIZE") or "16GB"
# The worker holds its own file read-write. Mirror versions are for the web workers only.
env["AIOS_MIRROR_VERSIONS"] = "0"
env["AIOS_UVICORN_WORKERS"] = "1"
@@ -129,6 +130,7 @@ def main(argv=None):
print(f"[sync-worker] pid={os.getpid()} tenant={tenant} "
f"scope={os.environ.get('AIOS_WORKER_SCOPE') or 'primary'} duckdb={worker_db} "
f"memory_limit={os.environ['AIOS_DUCKDB_MEMORY']} threads={os.environ['AIOS_DUCKDB_THREADS']} "
+ f"spill_limit={os.environ['AIOS_DUCKDB_MAX_TEMP_SIZE']} "
f"nice={niceness} job={mode or 'none'}",
flush=True)
print("[sync-worker] amazon_store.sync includes Brand Analytics reports in the worker role", flush=True)
diff --git a/api/test_odoo_rank_refresh.py b/api/test_odoo_rank_refresh.py
index bec64cc86b290c1daed3d258895e46a07f59d62a..7561e0247fa186be2851022eb3076ddfd34fa0ff 100644
--- a/api/test_odoo_rank_refresh.py
+++ b/api/test_odoo_rank_refresh.py
@@ -353,7 +353,7 @@ class RankRefreshTests(unittest.TestCase):
# own duration (hotfix 2026-09-29), so a clock that jumped a whole cadence inside
# every pass would ask for a zero wait.
clock = SimpleNamespace(sleep=sleep, time=lambda: 100 + cycle[0] * 60,
- strftime=lambda *_: 'synthetic-time', gmtime=lambda: ())
+ strftime=lambda *_: 'synthetic-time', gmtime=lambda *_: ())
ns = dict(os=os, _restore_sync_request_desks=lambda: None, RESYNC_STATE={'consecutiveFailures': 0}, _RESYNC_NOTICE=[None], _SEED_THREAD=None,
_configured_mirror_entities=lambda *_: None, freeze_notice=FREEZE_OFF,
_resolve_resync_cadence=lambda: dict(resolvedSeconds=60, manual=False,
@@ -675,7 +675,7 @@ class RankRefreshTests(unittest.TestCase):
return rebuild_ok
clock = SimpleNamespace(sleep=sleep, time=lambda: now,
- strftime=lambda *_: 'synthetic-time', gmtime=lambda: ())
+ strftime=lambda *_: 'synthetic-time', gmtime=lambda *_: ())
ns = dict(os=os, _restore_sync_request_desks=lambda: None, RESYNC_STATE={'consecutiveFailures': 0}, _RESYNC_NOTICE=[None], _SEED_THREAD=None,
_configured_mirror_entities=lambda *_: None,
freeze_notice=SimpleNamespace(enabled=lambda: frozen),
@@ -957,7 +957,7 @@ class RankRefreshTests(unittest.TestCase):
if status is not None:
store.docs[rel._ut().ODOO_SYNC_STATUS_KEY] = dict(status)
clock = SimpleNamespace(sleep=wait, monotonic=lambda: now[0], time=lambda: now[0],
- strftime=lambda *_: 'synthetic-time', gmtime=lambda: ())
+ strftime=lambda *_: 'synthetic-time', gmtime=lambda *_: ())
ns = dict(os=os, _restore_sync_request_desks=lambda: None, RESYNC_STATE={'consecutiveFailures': 0}, _RESYNC_NOTICE=[None], _SEED_THREAD=None,
_configured_mirror_entities=lambda *_: None, freeze_notice=FREEZE_OFF,
_resolve_resync_cadence=lambda: dict(
diff --git a/api/tool_groups/apps.py b/api/tool_groups/apps.py
index 443f14c988adbe9913c92da1bfdf30442242ad70..1705bf2e0317c732c0398639878535cd7fcae447 100644
--- a/api/tool_groups/apps.py
+++ b/api/tool_groups/apps.py
@@ -470,7 +470,15 @@ def apply_template(ctx, args):
allowed_databases=allowed, semantic_records=records,
field_contracts=_fields(ctx, allowed), **attribution)
return _result(result)
- except (applications.ApplicationError, PermissionError) as exc:
+ except applications.ApplicationError as exc:
+ # Keep typed refusals and inaccessible-App responses on their original path.
+ if type(exc) is applications.ApplicationError and "does not exist" not in str(exc):
+ kind = json.dumps(args["templateKind"], ensure_ascii=True)
+ exc = applications.ApplicationError(
+ f"Cannot apply templateKind {kind}. {str(exc).rstrip('.')}. "
+ "Check this template's inputs with apps.templates and retry.")
+ _refuse(exc)
+ except PermissionError as exc:
_refuse(exc)
diff --git a/platform/core/activity_log.py b/platform/core/activity_log.py
index 5836c5a607de954d84c3e3f2137868d9be0306dc..d1719094ee2000d041272d39bb9f3a67aaf2dee1 100644
--- a/platform/core/activity_log.py
+++ b/platform/core/activity_log.py
@@ -12,7 +12,7 @@ import re
DATABASE_KEY = 'ut_activity'
SYSTEM_KIND = 'activity'
-KINDS = frozenset(('record', 'field', 'cell', 'file', 'app', 'view'))
+KINDS = frozenset(('record', 'field', 'cell', 'file', 'app', 'view', 'view_as'))
OBJECT_TYPES = frozenset(('database', 'workbook', 'file', 'app'))
CHANGES_MAX_BYTES = 4096
RETENTION_MONTHS = 13
@@ -242,7 +242,7 @@ def seal_copy_batch(rt, events, outbox, *, backup_path, definitions=None):
def append(rt, *, kind, object, by, via=None, on_behalf_of=None, changes=None, _delivery=None):
"""Append one C5 event and remove at most PRUNE_BATCH expired events in the same transaction."""
if kind not in KINDS:
- raise ValueError('kind must be record, field, cell, file, app or view.')
+ raise ValueError('kind must be record, field, cell, file, app, view or view_as.')
object_type, object_key, object_id = _object(object)
by = _text(by, 'by')
via = _text(via, 'via', optional=True)
@@ -522,12 +522,13 @@ class PgActivityStore:
needed = {f'{self.name}_{suffix}' for suffix in (
'delivery', 'time', 'object_time', 'object_id_time', 'person_time')}
if (set(_COLUMNS) | {'delivery_key', 'delivery_version'} <= {r[0] for r in columns}
- and needed <= {r[0] for r in indexes} and constraint and "'view'" in constraint[0]):
+ and needed <= {r[0] for r in indexes} and constraint
+ and "'view_as'" in constraint[0]):
return
con.execute('SELECT pg_advisory_xact_lock(hashtextextended(%s, 0))', (self.table,))
con.execute(f'''CREATE TABLE IF NOT EXISTS {self.table} (
id BIGSERIAL PRIMARY KEY, occurred_at TIMESTAMPTZ NOT NULL,
- kind TEXT NOT NULL CHECK (kind IN ('record','field','cell','file','app','view')),
+ kind TEXT NOT NULL CHECK (kind IN ('record','field','cell','file','app','view','view_as')),
object_type TEXT NOT NULL, object_key TEXT NOT NULL, object_id TEXT NOT NULL,
actor TEXT NOT NULL, via TEXT, on_behalf_of TEXT,
changes JSONB NOT NULL, changes_truncated BOOLEAN NOT NULL,
@@ -541,11 +542,11 @@ class PgActivityStore:
'SELECT pg_get_constraintdef(oid) FROM pg_constraint '
'WHERE conrelid = %s::regclass AND conname = %s',
(self.table, constraint)).fetchone()
- if prior is None or "'view'" not in prior[0]:
+ if prior is None or "'view_as'" not in prior[0]:
con.execute(f'ALTER TABLE {self.table} DROP CONSTRAINT IF EXISTS "{constraint}"')
# The former constraint already admitted every existing event kind.
con.execute(f'ALTER TABLE {self.table} ADD CONSTRAINT "{constraint}" '
- "CHECK (kind IN ('record','field','cell','file','app','view')) NOT VALID")
+ "CHECK (kind IN ('record','field','cell','file','app','view','view_as')) NOT VALID")
for suffix, columns in (
('time', 'occurred_at DESC, id DESC'),
('object_time', 'object_type, object_key, occurred_at DESC, id DESC'),
diff --git a/platform/core/amazon_project_results.py b/platform/core/amazon_project_results.py
index da529b7db92725bfe49e237360ce6587f51fb088..8276e5e66a96ead2fe731511b9ab37fbb8aa9cf7 100644
--- a/platform/core/amazon_project_results.py
+++ b/platform/core/amazon_project_results.py
@@ -1,4 +1,4 @@
-"""Typed project result configurations and tenant-day completion windows."""
+"""Typed project results and daily series with independent source reconciliation."""
from datetime import date, timedelta
import hashlib
@@ -13,6 +13,121 @@ METRICS = ('sales', 'units', 'returns', 'return_rate', 'refund_rate', 'roas')
GRAINS = ('sku', 'asin', 'parent_asin')
MAX_CONFIGS = 20
BOUNDARY = '[completedOn, endExclusive) in the tenant timezone. Today is excluded.'
+SERIES_METRICS = (
+ ('sales', 'Sales', 'currency'), ('units', 'Units', 'units'),
+ ('margin', 'Margin', 'percent'), ('returns', 'Returns', 'units'),
+ ('returns_pct', 'Returns %', 'percent'), ('refunds_pct', 'Refunds %', 'percent'),
+ ('roas', 'ROAS', 'ratio'), ('sales_yoy', 'Sales YoY', 'percent'),
+)
+SERIES_BOUNDARY = 'Inclusive requested dates in the tenant timezone. Today is incomplete and excluded from totals.'
+
+
+class ValidationFailed(RuntimeError):
+ """A daily project series does not reconcile to its independent source aggregate."""
+
+
+def series_scope(value='all'):
+ """Parse the modal filter without allowing it to widen project membership."""
+ if value == 'all':
+ return None
+ if not isinstance(value, str) or ':' not in value:
+ raise ValueError('scope must be all, sku:, asin: or parent:.')
+ grain, text = value.split(':', 1)
+ keys = [key.strip() for key in text.split(',')]
+ if (grain not in ('sku', 'asin', 'parent') or not all(keys)
+ or (grain != 'sku' and len(keys) != 1)):
+ raise ValueError('scope must be all, sku:, asin: or parent:.')
+ return {'grain': grain, 'keys': list(dict.fromkeys(keys))}
+
+
+def series_window(project, start=None, end=None, *, today=None):
+ """Default to thirty days before start through today and aggregate only completed tenant days."""
+ today = today or periods.today()
+
+ def parsed(value):
+ if not isinstance(value, str) or not re.fullmatch(r'\d{4}-\d{2}-\d{2}', value):
+ raise ValueError('Series dates must use YYYY-MM-DD.')
+ return date.fromisoformat(value)
+
+ first = (parsed(start) if start is not None else
+ parsed(project.get('start_on') or today.isoformat()) - timedelta(days=30))
+ last = parsed(end) if end is not None else today
+ if first > last or last > today:
+ raise ValueError('Series dates must be ordered and cannot end after today.')
+ stop = min(last + timedelta(days=1), today)
+ lo, hi, rule = periods.yoy_window(first.isoformat(), stop.isoformat(), 'day')
+ return {'from': first.isoformat(), 'to': last.isoformat(), 'days': (stop - first).days,
+ 'current': (first, stop), 'comparison': (date.fromisoformat(lo), date.fromisoformat(hi)),
+ 'compare_rule': rule}
+
+
+def _finite(value):
+ return isinstance(value, (int, float)) and not isinstance(value, bool) and math.isfinite(value)
+
+
+def validate(answer, *, independent):
+ """Requery independent totals and return failing checks for any changed or unproved series."""
+ checks = []
+ first, last = date.fromisoformat(answer['from']), date.fromisoformat(answer['to'])
+ dates = [(first + timedelta(days=i)).isoformat() for i in range((last - first).days + 1)]
+ expected_keys = [key for key, _label, _unit in SERIES_METRICS]
+ metrics = answer['metrics']
+ if [metric['key'] for metric in metrics] != expected_keys:
+ return [{'check': 'series total ties to economics', 'ok': False,
+ 'reason': 'Series metrics do not follow the project result contract.'}]
+ for metric in metrics:
+ ours = metric.get('total')
+ source = independent(metric['key'])
+ expected, source_reason = source['value'], source.get('reason')
+ points = metric.get('points', [])
+ ok = [point.get('date') for point in points] == dates
+ if ours is None:
+ ok = ok and bool(metric.get('reason')) and all(point.get('value') is None for point in points)
+ else:
+ ok = (ok and _finite(ours) and _finite(expected)
+ and math.isclose(ours, expected, rel_tol=1e-9, abs_tol=1e-6)
+ and all(point.get('value') is None or _finite(point['value']) for point in points))
+ if metric['key'] in ('sales', 'units', 'returns'):
+ values = [point['value'] for point in points if point.get('value') is not None]
+ ok = ok and bool(values) and math.isclose(math.fsum(values), ours, rel_tol=1e-9, abs_tol=1e-6)
+ tie = metric.get('tie', {})
+ if tie.get('ok') or tie.get('ours') is not None or tie.get('independent') is not None:
+ ok = (ok and ours is not None and tie.get('ok') is True
+ and _finite(tie.get('ours')) and _finite(tie.get('independent'))
+ and math.isclose(tie['ours'], ours, rel_tol=1e-9, abs_tol=1e-6)
+ and _finite(expected)
+ and math.isclose(tie['independent'], expected, rel_tol=1e-9, abs_tol=1e-6))
+ checks.append({'check': 'series total ties to economics', 'metric': metric['key'], 'ok': bool(ok),
+ 'ours': ours, 'independent': expected,
+ 'reason': metric.get('reason') or source_reason})
+ return checks
+
+
+def series(project, *, bounds, read, independent):
+ """Publish the fixed chart order and project markers after independent economics reconciliation."""
+ first, last = date.fromisoformat(bounds['from']), date.fromisoformat(bounds['to'])
+ dates = [(first + timedelta(days=i)).isoformat() for i in range((last - first).days + 1)]
+ answer = {'projectId': project['id'], 'from': bounds['from'], 'to': bounds['to'],
+ 'timezone': timezone_name(), 'boundary': SERIES_BOUNDARY,
+ 'markers': {'start': project.get('start_on'), 'target': project.get('target_on'),
+ 'completed': project.get('completed_on')}, 'metrics': []}
+ for key, label, unit in SERIES_METRICS:
+ measured = read(key)
+ total, reason = measured.get('total'), measured.get('reason')
+ if total is None and not reason:
+ reason = 'An independent daily source for this metric is unavailable.'
+ values = {point['date']: point['value'] for point in measured.get('points', [])} if total is not None else {}
+ answer['metrics'].append({'key': key, 'label': label, 'unit': measured.get('unit', unit),
+ 'points': [{'date': day, 'value': values.get(day)} for day in dates],
+ 'total': total, 'reason': reason,
+ 'tie': {'ok': False, 'ours': None, 'independent': None}})
+ checks = validate(answer, independent=independent)
+ if any(not check['ok'] for check in checks):
+ raise ValidationFailed('A series total does not tie to its independent Amazon economics aggregate.')
+ for metric, check in zip(answer['metrics'], checks):
+ if metric['total'] is not None:
+ metric['tie'] = {'ok': True, 'ours': check['ours'], 'independent': check['independent']}
+ return answer
def configurations(values):
diff --git a/platform/core/amazon_projects.py b/platform/core/amazon_projects.py
index ba62b9bb31ae7eab8c5c749d9e3e49f40beaed3c..6004b0298213b0752ee8b2d9df37dd7e33b4d512 100644
--- a/platform/core/amazon_projects.py
+++ b/platform/core/amazon_projects.py
@@ -250,10 +250,11 @@ def health(project, today):
def summary(projects):
- """R19 and R67: average progress and "N of M completed", with Canceled left out of M."""
+ """Average progress with Completed at 100 and Canceled excluded."""
counted = [p for p in projects if p.get('status') != CANCELED]
total = len(counted)
- return {'average': round(sum(p['progress'] for p in counted) / total) if total else None,
+ return {'average': round(sum(100 if p.get('status') == COMPLETED else p['progress']
+ for p in counted) / total) if total else None,
'completed': sum(1 for p in counted if p.get('status') == COMPLETED), 'total': total}
diff --git a/platform/core/connector_sync.py b/platform/core/connector_sync.py
index 5c83fb151e9f1f0f273b83b24407d14488d19b06..2cb3c1e6ef9235b150f1560bf79abe33586b38a3 100644
--- a/platform/core/connector_sync.py
+++ b/platform/core/connector_sync.py
@@ -162,6 +162,23 @@ def sync_not_before(provider, tenant):
return _SYNC_NOT_BEFORE.get((provider, tenant))
+def _clear_sync_defer(provider, tenant):
+ """Remove one obsolete provider hold under the desk lock, retaining explicit continuation waits."""
+ key = (provider, tenant)
+ held = _SYNC_NOT_BEFORE.pop(key, None)
+ receipt = _SYNC_REQUESTS.get(key)
+ if (held is not None and receipt and receipt["state"] == "queued"
+ and receipt.get("requestedBy") != "first load" and receipt.get("notBefore") == held):
+ receipt["notBefore"] = None
+
+
+def clear_sync_defer(provider, tenant):
+ """Release one provider hold after its producer adopts partition-specific waits."""
+ with _SYNC_REQUEST_LOCK:
+ _clear_sync_defer(provider, tenant)
+ _wake_sync_requests(time.time())
+
+
def register_request_listener(fn):
"""Call fn(receipt, connection_id) after each accepted ordinary request, outside the desk lock."""
if fn not in _REQUEST_LISTENERS:
@@ -221,7 +238,13 @@ def _request_sync(provider, tenant, *, requested_by="", now=None, not_before=Non
_wake_sync_requests(stamp)
return {**result, "accepted": accepted_id == identity, "duplicate": accepted_id != identity}
with _SYNC_REQUEST_LOCK:
+ if provider == "amazon_sp" and requested_by != "first load":
+ # Amazon's history floor belongs to owed tables, never a loaded-table refresh.
+ _clear_sync_defer(provider, tenant)
previous = _SYNC_REQUESTS.get((provider, tenant))
+ replaces_amazon_history = (provider == "amazon_sp" and previous
+ and previous.get("requestedBy") == "first load"
+ and requested_by != "first load")
if previous and previous["state"] in ("queued", "running"):
if not (previous["state"] == "queued" and previous.get("requestedBy") == "first load"
and requested_by != "first load"):
@@ -230,7 +253,7 @@ def _request_sync(provider, tenant, *, requested_by="", now=None, not_before=Non
"requestedAt": time.time() if now is None else now,
"notBefore": max(float(not_before or 0), _SYNC_NOT_BEFORE.get((provider, tenant), 0),
float(previous.get("notBefore") or 0) if previous
- and previous["state"] == "queued" else 0) or None,
+ and previous["state"] == "queued" and not replaces_amazon_history else 0) or None,
"startedAt": None, "finishedAt": None, "detail": "",
"requestedBy": str(requested_by or ""), "accepted": True}
if requested_by == "first load" and previous and previous.get("firstLoad"):
@@ -471,6 +494,9 @@ def poll_remote_requests(rt, *, now=None):
if stamp - float(entry.get("finishedAt") or 0) > REMOTE_KEEP_SECONDS:
pruned.append(identity)
continue
+ manual_amazon = key[0] == "amazon_sp" and entry.get("requestedBy") != "first load"
+ if manual_amazon:
+ _clear_sync_defer(*key)
receipt = _SYNC_REQUESTS.get(key)
# Promote under the same lock as claims, so a continuation cannot win a manual hand-off race.
if (receipt and receipt["state"] == "queued" and receipt.get("requestedBy") == "first load"
@@ -478,7 +504,7 @@ def poll_remote_requests(rt, *, now=None):
receipt.update(id=identity, requestedBy=entry.get("requestedBy") or "",
requestedAt=entry["requestedAt"],
connectionId=entry.get("connectionId"),
- notBefore=max(float(receipt.get("notBefore") or 0),
+ notBefore=max(0 if manual_amazon else float(receipt.get("notBefore") or 0),
float(entry.get("notBefore") or 0),
_SYNC_NOT_BEFORE.get(key, 0)) or None)
admitted.append(deepcopy(entry))
diff --git a/platform/core/external_access.py b/platform/core/external_access.py
index 66f143a4ca730a8014d8c6e4314be3a1dffb443c..686f038fd3f02fd86f9c53c4f295342494d67a7c 100644
--- a/platform/core/external_access.py
+++ b/platform/core/external_access.py
@@ -11,6 +11,8 @@ from dataclasses import dataclass
from datetime import datetime, timezone
import hashlib
import hmac
+import logging
+import os
import secrets
import time
@@ -18,6 +20,7 @@ import core.users as users
from harness import runtime
+_log = logging.getLogger(__name__)
BUCKET = "external_access"
ACCESS_READ = "read"
ACCESS_READWRITE = "readwrite"
@@ -57,6 +60,10 @@ class ExternalAccessError(RuntimeError):
class ExternalAuthError(ExternalAccessError):
"""A uniform authentication refusal that reveals no credential or tenant detail."""
+ def __init__(self, code: str, *, cause: str = "unknown_key"):
+ super().__init__(code)
+ self.cause = cause
+
def normalise_access(value, *, allow_refresh=False) -> str:
if value is None or value == "":
@@ -139,6 +146,8 @@ class VerifiedCredential:
access: str = ACCESS_READWRITE
databases: object = ALL_DATABASES
workbooks: tuple = ()
+ warning: str | None = None
+ origin_recorded: bool = False
def _now() -> str:
@@ -329,6 +338,7 @@ def mint(st, username: str, tenant: str, label: str, access=None, databases=None
"rotated_at": None,
"expires_at": expiry,
"test_connection": test_connection,
+ "minted_space": os.environ.get("SPACE_ID") or None,
}
book[key_id] = record
minted["record"] = record
@@ -424,6 +434,7 @@ def rotate(st, username: str, tenant: str, key_id: str) -> tuple[dict, str]:
record.update({
"salt": salt.hex(), "hash": _digest(salt, secret), "algo": ALGORITHM,
"rotated_at": _now(), "last_used": None,
+ "minted_space": os.environ.get("SPACE_ID") or None,
})
book[target] = record
rotated["record"] = record
@@ -514,31 +525,63 @@ def _comparison_material(record) -> tuple[bool, bytes, str]:
return False, _DUMMY_SALT, _DUMMY_HASH
-def verify(token: str) -> VerifiedCredential:
+def _recorded_space(record: dict) -> str | None:
+ """Treat missing or malformed origin metadata as unproved."""
+ value = record.get("minted_space")
+ return value if isinstance(value, str) and value.strip() else None
+
+
+def _credential_cause(record, tenant: str, key_id: str, secret: str, *, owner=None):
+ """Inspect status only after the secret and optional session owner match the stored record."""
+ material_ok, salt, expected = _comparison_material(record)
+ matched = hmac.compare_digest(_digest(salt, secret), expected)
+ if (not material_ok or not matched or record.get("id") != key_id
+ or record.get("tenant") != tenant or not record.get("username")
+ or (owner is not None and record.get("username") != owner)):
+ return "unknown_key"
+ origin, current = _recorded_space(record), os.environ.get("SPACE_ID")
+ if origin and current and origin != current:
+ return "minted_elsewhere"
+ if record.get("revoked_at"):
+ return "revoked"
+ if record.get("expires_at"):
+ expiry = _epoch(record["expires_at"])
+ if expiry is None or expiry <= time.time():
+ return "expired"
+ return None
+
+
+def verify_for_owner(st, username: str, tenant: str, token: str) -> dict:
+ """Diagnose only a session owner's secret inside that session's tenant runtime."""
+ owner, tenant_key = _normalise_owner(username, tenant)
+ _assert_bound_runtime(st, tenant_key)
+ try:
+ verified = verify(token, owner_runtime=st, owner=owner)
+ except ExternalAuthError as exc:
+ return {"valid": False, "cause": exc.cause, "warning": None, "origin_recorded": None}
+ return {"valid": True, "cause": None, "warning": verified.warning,
+ "origin_recorded": verified.origin_recorded, "key_id": verified.key_id}
+
+
+def verify(token: str, *, owner_runtime=None, owner=None) -> VerifiedCredential:
"""Verify a bearer token from fresh state and resolve its ordinary public-user principal."""
try:
tenant_key, key_id, secret = _parse_token(token)
- try:
- st = runtime.get_runtime(tenant_key)
- except KeyError as exc:
- raise ExternalAuthError(AUTH_ERROR) from exc
+ if owner_runtime is not None:
+ if not owner or getattr(owner_runtime, "key", None) != tenant_key:
+ raise ExternalAuthError(AUTH_ERROR)
+ st = owner_runtime
+ else:
+ try:
+ st = runtime.get_runtime(tenant_key)
+ except KeyError as exc:
+ raise ExternalAuthError(AUTH_ERROR) from exc
book = _book(st)
record = book.get(key_id)
- material_ok, salt, expected = _comparison_material(record)
- matched = hmac.compare_digest(_digest(salt, secret), expected)
- record_ok = (
- material_ok
- and isinstance(record, dict)
- and record.get("id") == key_id
- and record.get("tenant") == tenant_key
- and bool(record.get("username"))
- and not record.get("revoked_at")
- and (_epoch(record.get("expires_at")) is None
- if not record.get("expires_at") else _epoch(record.get("expires_at")) > time.time())
- )
- if not record_ok or not matched:
- raise ExternalAuthError(AUTH_ERROR)
+ cause = _credential_cause(record, tenant_key, key_id, secret, owner=owner)
+ if cause:
+ raise ExternalAuthError(AUTH_ERROR, cause=cause)
owner = str(record["username"]).strip().lower()
try:
@@ -549,7 +592,8 @@ def verify(token: str) -> VerifiedCredential:
access, databases = scope_of(record)
if not _stamp_is_due(record.get("last_used"), time.time()):
return VerifiedCredential(tenant=tenant_key, key_id=key_id, user=public, runtime=st,
- access=access, databases=databases, workbooks=workbooks_of(record))
+ access=access, databases=databases, workbooks=workbooks_of(record),
+ origin_recorded=bool(_recorded_space(record)))
used_at = _now()
@@ -557,26 +601,32 @@ def verify(token: str) -> VerifiedCredential:
if not isinstance(value, dict):
raise ExternalAuthError(AUTH_ERROR)
current = value.get(key_id)
- if (not isinstance(current, dict) or current.get("revoked_at")
- or current.get("username") != owner
- or current.get("tenant") != tenant_key
- or current.get("hash") != record.get("hash")):
- raise ExternalAuthError(AUTH_ERROR)
+ cause = _credential_cause(current, tenant_key, key_id, secret, owner=owner)
+ if cause:
+ raise ExternalAuthError(AUTH_ERROR, cause=cause)
book_now = dict(value)
stamped = dict(current)
stamped["last_used"] = used_at
book_now[key_id] = stamped
return book_now
+ warning = None
try:
st.update(BUCKET, _stamp, flush="sync")
except ExternalAuthError:
raise
except Exception as exc:
- raise ExternalAuthError(AUTH_ERROR) from exc
+ # Store diagnostics must not include credential material or exception text.
+ _log.warning("external_access last_used stamp failed: %s", type(exc).__name__)
+ warning = "store_write_refused"
return VerifiedCredential(tenant=tenant_key, key_id=key_id, user=public, runtime=st,
- access=access, databases=databases, workbooks=workbooks_of(record))
+ access=access, databases=databases, workbooks=workbooks_of(record),
+ warning=warning, origin_recorded=bool(_recorded_space(record)))
except ExternalAuthError:
raise
+ except ExternalAccessError as exc:
+ if owner_runtime is not None:
+ raise
+ raise ExternalAuthError(AUTH_ERROR) from exc
except Exception as exc:
raise ExternalAuthError(AUTH_ERROR) from exc
diff --git a/platform/core/table_store.py b/platform/core/table_store.py
index 34cb57e247a4acb4623de89ca50bb0d10bad9800..6073dc3464a6e7415032fb23ab2ad80c342c8de8 100644
--- a/platform/core/table_store.py
+++ b/platform/core/table_store.py
@@ -105,10 +105,9 @@ TRASH_RETENTION_SECONDS = TRASH_RETENTION_DAYS * 24 * 60 * 60
# browser re-reads only after some writes. So "stamp differs" alone refuses one tab's own second
# write, and the outbox then resends that refused write forever.
#
-# The record below tells the two apart. Each view write appends the view's member digests as
-# they stood BEFORE it, tagged with the revision it landed as. For a stamp `R`, the earliest
-# entry after `R` is the view as that sender saw it. A write is refused only when a member
-# changed after `R` and this write would set it to something else.
+# Each view write records its author, landed revision and the member digests from before it.
+# The next before-image isolates each write's changes. Known requester writes are skipped.
+# A foreign or unknown writer's changed member stays protected when this write would replace it.
#
# ⛔ The record lives in the owner's own stratum beside `views`, so it rides the same update and
# the same settings row. `workspace()` never projects it.
@@ -121,7 +120,7 @@ TRASH_RETENTION_SECONDS = TRASH_RETENTION_DAYS * 24 * 60 * 60
VIEW_WRITES_KEY = 'viewWrites'
# Folder ids are surface-scoped and keep history separate from view ids.
FOLDER_WRITES_KEY = 'folderWrites'
-#: Writes kept per view. A sender older than the oldest kept write cannot be judged and is refused.
+#: Writes kept per view. Older senders need a matching trimmed-author boundary.
VIEW_WRITES_KEEP = 8
#: Views kept per stratum. The least recently written record goes first.
VIEW_WRITES_VIEWS = 400
@@ -150,7 +149,7 @@ def view_members(view):
return out
-def _note_write(ws, view_id, at, kind, before, book_key):
+def _note_write(ws, view_id, at, kind, before, book_key, *, author=None):
"""Append one write to the selected bounded history in this owner's stratum."""
if at is None or not view_id or not isinstance(ws, dict):
return
@@ -159,11 +158,21 @@ def _note_write(ws, view_id, at, kind, before, book_key):
book = ws[book_key] = {}
entry = book.get(view_id) if isinstance(book.get(view_id), dict) else {}
log = [row for row in entry.get('log') or () if isinstance(row, dict)]
- log.append({'rev': int(at), 'kind': kind, 'before': before})
+ author = str(author or '').strip().lower() or None
+ log.append({'rev': int(at), 'kind': kind, 'before': before, 'author': author})
since = int(entry.get('since') or 0)
+ trimmed = entry.get('trimmed')
+ trimmed = dict(trimmed) if isinstance(trimmed, dict) else {}
while len(log) > VIEW_WRITES_KEEP:
- since = max(since, int(log.pop(0).get('rev') or 0))
+ removed = log.pop(0)
+ removed_author = removed.get('author')
+ # Only a known, uninterrupted author run can excuse writes outside the ring.
+ if not removed_author or removed_author != trimmed.get('author'):
+ trimmed = {'author': removed_author, 'after': since}
+ since = max(since, int(removed.get('rev') or 0))
book[view_id] = {'since': since, 'log': log}
+ if trimmed:
+ book[view_id]['trimmed'] = trimmed
if len(book) > VIEW_WRITES_VIEWS:
# jsonb keeps no key order, so recency is read off each record's newest revision.
def newest(vid):
@@ -174,9 +183,9 @@ def _note_write(ws, view_id, at, kind, before, book_key):
book.pop(vid, None)
-def note_view_write(ws, view_id, at, kind, before):
+def note_view_write(ws, view_id, at, kind, before, *, author=None):
"""Append one view, gone or place write to this stratum's record."""
- _note_write(ws, view_id, at, kind, before, VIEW_WRITES_KEY)
+ _note_write(ws, view_id, at, kind, before, VIEW_WRITES_KEY, author=author)
def _later_writes(ws, view_id, sent, kinds, *, book_key=VIEW_WRITES_KEY):
@@ -191,25 +200,31 @@ def _later_writes(ws, view_id, sent, kinds, *, book_key=VIEW_WRITES_KEY):
return later, int(sent) >= int(entry.get('since') or 0)
-def stale_view_conflict(ws, view_id, mine, sent):
+def stale_view_conflict(ws, view_id, mine, sent, *, requester=None):
"""Whether saving `mine` over `view_id` undoes a change made after revision `sent`."""
later, judgeable = _later_writes(ws, view_id, sent, ('view', 'gone'))
- if not later:
- return False
+ requester = str(requester or '').strip().lower() or None
if not judgeable:
- return True
- if later[0].get('kind') == 'view' and later[0].get('before') is None:
- # Created after `sent`, so the sender is the tab that created it. Judge only later writes.
- later = later[1:]
- if not later:
- return False
+ entry = (ws.get(VIEW_WRITES_KEY) or {}).get(view_id) or {}
+ trimmed = entry.get('trimmed')
+ trimmed = trimmed if isinstance(trimmed, dict) else {}
+ if not (requester and trimmed.get('author') == requester
+ and int(sent) >= int(trimmed.get('after', entry.get('since') or 0))):
+ return True
current = ((ws.get('views') or {}).get(view_id))
- if not isinstance(current, dict):
- return True
- base = later[0].get('before') or {}
- now, want = view_members(current), view_members(mine) or {}
- return any(now.get(member) != base.get(member) and want.get(member) != now.get(member)
- for member in set(base) | set(now))
+ now, want = view_members(current) or {}, view_members(mine) or {}
+ for index, row in enumerate(later):
+ if requester and row.get('author') == requester:
+ continue
+ if not isinstance(current, dict):
+ return True
+ before = row.get('before') or {}
+ after = (later[index + 1].get('before') or {}) if index + 1 < len(later) else now
+ # The next before-image isolates this writer's changes from the requester's later edits.
+ if any(before.get(member) != after.get(member) and want.get(member) != now.get(member)
+ for member in set(before) | set(after)):
+ return True
+ return False
def stale_place_conflict(ws, view_id, folder, sent):
@@ -1391,14 +1406,15 @@ class TableStore:
for folder_id in sorted(set(before_parents) | set(parents)):
if before_parents.get(folder_id) != parents.get(folder_id):
_note_write(ws, folder_id, landing.get('at'), 'parent',
- {'parent': before_parents.get(folder_id)}, FOLDER_WRITES_KEY)
+ {'parent': before_parents.get(folder_id)}, FOLDER_WRITES_KEY,
+ author=actor or username)
ws['folders'] = tree
ws['itemFolders'] = placed
now = placed.get('views') or {}
for view_id in sorted(set(was) | {str(k) for k in now}):
if was.get(view_id) != now.get(view_id):
note_view_write(ws, view_id, landing.get('at'), 'place',
- {'folderId': was.get(view_id)})
+ {'folderId': was.get(view_id)}, author=actor or username)
def _audit(ws, record):
now = (ws.get('itemFolders') or {}).get('views') or {}
@@ -1994,7 +2010,8 @@ class TableStore:
prior = (ws.get('views') or {}).get(view_id)
activity_before['value'] = view_audit.summary(prior)
ws.setdefault('views', {})[view_id] = payload
- note_view_write(ws, view_id, landing.get('at'), 'view', view_members(prior))
+ note_view_write(ws, view_id, landing.get('at'), 'view', view_members(prior),
+ author=actor or username)
audit(action=action or view_audit.derive_action(prior, payload),
owner=username, view_id=view_id, actor=actor, origin=origin,
before=view_audit.summary(prior), after=view_audit.summary(payload))
@@ -2010,7 +2027,8 @@ class TableStore:
if rev is None or not rebase:
raise
self._land_stale(username, rev, stale,
- lambda ws, sent: stale_view_conflict(ws, view_id, requested, sent),
+ lambda ws, sent: stale_view_conflict(
+ ws, view_id, requested, sent, requester=actor or username),
_write)
self.log_view_activity(view_id, 'saved', by=actor or username,
before=activity_before.get('value'),
@@ -2055,7 +2073,8 @@ class TableStore:
removed.clear()
if isinstance(before, dict):
removed['value'] = view_audit.summary(before)
- note_view_write(data.setdefault(owner, {}), vid, at, 'gone', view_members(before))
+ note_view_write(data.setdefault(owner, {}), vid, at, 'gone', view_members(before),
+ author=who)
views.pop(vid, None)
audit(action='delete' if isinstance(before, dict) else None,
owner=owner, view_id=vid, actor=actor, origin=origin,
diff --git a/platform/core/user_tables.py b/platform/core/user_tables.py
index c1d61a5a5cc75f348af334bb3a87011f94b4201f..92d11fbd4c11a4bc98ee7458ea6e3ff14b78c20c 100644
--- a/platform/core/user_tables.py
+++ b/platform/core/user_tables.py
@@ -529,6 +529,8 @@ def oversize_of(table_key, st=None, defn=None):
defn = get_definition(str(table_key or '').strip(), st) or {}
except Exception: # noqa: BLE001
defn = {}
+ if not materialises(table_key, defn=defn):
+ return None
stamp = (defn or {}).get(OVERSIZE_FIELD)
return stamp if isinstance(stamp, dict) else None
diff --git a/platform/core/users.py b/platform/core/users.py
index e30968c5bb10535e4bdf0a92d840c1b48197c373..74a94c6c3ae7671d13ee5ec164d7f6f965ac56eb 100644
--- a/platform/core/users.py
+++ b/platform/core/users.py
@@ -1176,8 +1176,6 @@ def set_whatsapp(username, value):
u = reg.get(username)
if not isinstance(u, dict):
raise ValueError('This account is not available.')
- # R42: a test account has no outbound channel, so no writer may add one.
- refuse_test_account_change(u, whatsapp=value)
if value is None:
u.pop('whatsapp', None)
else:
@@ -1200,110 +1198,9 @@ def set_whatsapp_opt_out(username, on):
return account_rows.update(username, _set, st=store)
-# ------------------------------------------------------------------ test accounts (S79-T63, R42)
-#: The record key that marks a copied test account: `{copied_from, created_by, created_at}`.
-TEST_ACCOUNT_KEY = 'test_account'
-
-
-class TestAccountRefused(ValueError):
- """A writer refused to give a test account an outbound channel or the admin role."""
-
- def __init__(self, code, message):
- self.code = code
- super().__init__(message)
-
-
-class TestAccountExists(ValueError):
- """The chosen test username is already a key in the registry."""
-
-
def is_test_account(record):
- """Whether this stored record is a copied test account."""
- return isinstance(record, dict) and isinstance(record.get(TEST_ACCOUNT_KEY), dict)
-
-
-def refuse_test_account_change(record, *, email=None, whatsapp=None, role=None):
- """Raise `TestAccountRefused` if a change would give a test account email, WhatsApp or admin."""
- if not is_test_account(record):
- return
- if email:
- raise TestAccountRefused('test_account_no_email',
- 'A test account cannot have an email address.')
- if whatsapp:
- raise TestAccountRefused('test_account_no_whatsapp',
- 'A test account cannot connect WhatsApp.')
- if role is not None and str(role).strip().lower() != 'user':
- raise TestAccountRefused('test_account_not_admin',
- 'A test account cannot be an administrator.')
-
-
-def create_test_account(username, pw, *, name, access, marker, tenant):
- """Write one test account in one update: `access` fields verbatim, no email, no WhatsApp.
-
- Raises `TestAccountExists` when the key is taken by any record other than this exact write.
- """
- import copy
- freeze_notice.check_write()
- original_username = (username or '').strip()
- username = original_username.lower()
- failures = password_failures(pw, username=username, name=name)
- if failures:
- raise PasswordPolicyError(failures)
- marker = dict(marker)
- salt = secrets.token_hex(16)
- digest = _hash(pw, salt)
-
- def _add(reg):
- prior = reg.get(username)
- if prior is not None:
- # A replayed closure finds its own landed write; anything else is a real collision.
- if isinstance(prior, dict) and prior.get(TEST_ACCOUNT_KEY) == marker:
- return reg
- raise TestAccountExists(username)
- rec = {'salt': salt, 'hash': digest, 'name': name, 'role': 'user',
- 'password_changed_at': _now_iso(),
- 'bus': copy.deepcopy(access.get('bus', 'all')), 'active': True,
- 'modules': copy.deepcopy(access.get('modules', 'all')),
- 'agent': access.get('agent') or None, 'email': None,
- 'tenant': str(tenant or 'royal-imports').strip().lower(),
- 'username_original': original_username, TEST_ACCOUNT_KEY: dict(marker)}
- # The permission block and its marker travel together or not at all (see `_record`).
- for key in ('perms', 'perms_v'):
- if key in access:
- rec[key] = copy.deepcopy(access[key])
- reg[username] = rec
- return reg
-
- account_rows.update(username, _add, st=store)
-
-
-def delete_test_account(username, *, tenant, marker=None):
- """Delete one test account record. Returns deleted, missing or not_test; never a real account.
-
- `marker`, when given, must equal the stored marker, so a rollback never removes a copy
- another administrator made under the same name.
- """
- freeze_notice.check_write()
- username = (username or '').strip().lower()
- tenant = str(tenant or '').strip().lower()
- outcome = ['missing']
-
- def _del(reg):
- outcome[0] = 'missing'
- rec = reg.get(username)
- if (not isinstance(rec, dict)
- or str(rec.get('tenant') or 'royal-imports').strip().lower() != tenant):
- return reg
- if not is_test_account(rec) or (marker is not None
- and rec.get(TEST_ACCOUNT_KEY) != dict(marker)):
- outcome[0] = 'not_test'
- return reg
- reg.pop(username, None)
- outcome[0] = 'deleted'
- return reg
-
- account_rows.update(username, _del, st=store)
- return outcome[0]
+ """Read the legacy marker until release confirms live has no copied accounts."""
+ return isinstance(record, dict) and isinstance(record.get('test_account'), dict)
#: The keys of one database entry that `set_database_access` writes.
@@ -1422,7 +1319,6 @@ def set_access(username, role=None, bus=None, modules=None, agent=None, email=No
def _set(reg):
u = reg.get(username)
if u:
- refuse_test_account_change(u, email=email, role=role)
if name is not None:
u['name'] = name
if role is not None:
@@ -1465,12 +1361,6 @@ def update_access_with_active_admin_guard(username, role=None, bus=None, modules
if not isinstance(current, dict):
outcome[0] = 'missing'
return reg
- try:
- refuse_test_account_change(current, email=email, role=role)
- except TestAccountRefused as exc:
- outcome[0] = exc.code
- return reg
-
# Build the whole candidate before touching reg. On a refused transition the original
# record remains byte-for-byte intact, including its epoch.
candidate = dict(current)
diff --git a/platform/core/windows.py b/platform/core/windows.py
index f65b63bf8191823723d836baf9e6c09c016cc265..fd7c4a6de5daaa9fb2bea1abec44210728a02693 100644
--- a/platform/core/windows.py
+++ b/platform/core/windows.py
@@ -286,7 +286,10 @@ def ending_bounds(kind, mode, value, today, step="month", offset=0, n=0, unit="m
"""Keep the kind's own start. An ending before it refuses instead of reversing bounds."""
if kind not in TO_DATE_KINDS:
return None
- end = resolve_ending(mode, value, today, step, offset, n, unit)
+ ending_today = today
+ if kind == "ytd_last_year" and mode not in ("fixed", "cell"):
+ ending_today = resolve({"kind": kind}, today)[1]
+ end = resolve_ending(mode, value, ending_today, step, offset, n, unit)
if end is None:
return None
start = resolve({"kind": kind}, today)[0]
diff --git a/platform/harness/amazon_client.py b/platform/harness/amazon_client.py
index 19493bb6e32c8b2b57036e3a9fe1c01eb4d57597..f66da1a5436131318798110e7f058f8b0732892d 100644
--- a/platform/harness/amazon_client.py
+++ b/platform/harness/amazon_client.py
@@ -416,7 +416,7 @@ REPORT_TYPES = {
},
"storage_fees": {
"report": "GET_FBA_STORAGE_FEE_CHARGES_DATA", "monthly": True,
- "min_interval_s": 30 * 86400,
+ "min_interval_s": 86400,
"validates_against": "Amazon document data-row count vs stored rows for its latest calendar month",
"role": "Amazon Fulfillment",
},
@@ -458,6 +458,7 @@ GROUP_A_TABLES = {
"report": "GET_FBA_REIMBURSEMENTS_DATA",
"role": "Amazon Fulfillment",
"min_interval_s": 14400,
+ "restatement_keys": ("reimbursement_id",),
"cadence_note": "Date windows; refresh no sooner than 14400 seconds.",
"validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
},
@@ -466,6 +467,7 @@ GROUP_A_TABLES = {
"report": "GET_FBA_FULFILLMENT_CUSTOMER_RETURNS_DATA",
"role": "Amazon Fulfillment",
"min_interval_s": 14400,
+ "restatement_keys": ("order_id", "sku", "fnsku", "asin", "return_date"),
"cadence_note": "Date windows; refresh no sooner than 14400 seconds.",
"validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
},
@@ -676,8 +678,8 @@ GROUP_A_TABLES = {
"label": "Amazon account health",
"report": "GET_V2_SELLER_PERFORMANCE_REPORT",
"role": "Selling Partner Insights",
- "min_interval_s": 86400,
- "cadence_note": "Current snapshot; refresh no sooner than 86400 seconds.",
+ "min_interval_s": 43200,
+ "cadence_note": "Current snapshot; refresh no sooner than 43200 seconds.",
"snapshot": True,
"json_root": "performanceMetrics",
"json_schema": "accountHealthReport-2020-11-18.json",
@@ -734,6 +736,7 @@ GROUP_A_TABLES = {
"report": "GET_FLAT_FILE_RETURNS_DATA_BY_RETURN_DATE",
"role": "Inventory and Order Tracking",
"min_interval_s": 14400,
+ "restatement_keys": ("order_id", "sku", "return_request_date"),
"cadence_note": "Date windows; refresh no sooner than 14400 seconds.",
"max_window_days": 60,
"validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
@@ -1015,6 +1018,40 @@ GROUP_A = {
GROUP_A.update({key: {"key": key, **spec} for key, spec in DECLARED_DATASETS.items()})
+# Source: AutoWiki/research/datadoe-sync-cadence.md. Unlisted tables keep the daily default.
+CADENCE = {
+ table: {"perDay": 1, "lookbackDays": 1, "intradayLookbackDays": 1, "period": "day"}
+ for table in GROUP_A
+}
+for _table, _spec in REPORT_TYPES.items():
+ if _spec.get("monthly") or _spec.get("period") == "MONTH":
+ CADENCE[_table].update(period="month", lookbackDays=60, intradayLookbackDays=60)
+ elif _spec.get("weekly") or _spec.get("min_interval_s", 0) >= 7 * 86400:
+ CADENCE[_table].update(period="week", lookbackDays=14, intradayLookbackDays=14)
+for _tables, _passes, _daily, _intraday in (
+ (("orders", "order_items"), 6, 8, 1),
+ (("settlements", "transactions"), 3, 8, 2),
+ (("listings", "inventory_planning"), 4, 1, 1),
+ (("account_health", "manage_inventory", "restock_recommendations"), 2, 1, 1),
+ (("pricing", "fba_fee_estimates", "referral_fees"), 1, 4, 4),
+ (("reimbursements", "long_term_storage_fees"), 1, 30, 30),
+ (("fba_customer_returns", "mfn_returns"), 1, 60, 60),
+ (("storage_fees",), 1, 1, 1),
+):
+ for _table in _tables:
+ CADENCE[_table].update(perDay=_passes, lookbackDays=_daily,
+ intradayLookbackDays=_intraday, period="day")
+
+# Ads uses namespaced mirror marker names, so its tables cannot collide with SP-API tables.
+from harness import amazon_ads_schema as _ads_schema
+
+CADENCE.update({
+ "ads_" + table: {"perDay": 4, "lookbackDays": 21, "intradayLookbackDays": 5, "period": "day"}
+ for table in _ads_schema.TABLE_SPECS
+})
+del _table, _tables, _spec, _passes, _daily, _intraday, _ads_schema
+
+
# Inventory section 2.5: immutable refusal even if a future report roster admits one.
EXCLUDED_REPORT_TYPES = frozenset({
"GET_ORDER_REPORT_DATA_SHIPPING", "GET_ORDER_REPORT_DATA_TAX", "GET_ORDER_REPORT_DATA_ALL",
@@ -3381,7 +3418,8 @@ def report_document(fields, report_type, start=None, end=None, options=None,
# a role-shaped refusal on screen and left the window to retry forever. An empty
# document lets the window complete and the connector detail name the empty period.
say(f" {report_type}: Amazon cancelled report {rid}; no data for the requested period")
- return "", {"reportId": rid, "reportDocumentId": None, "cancelled": True,
+ return "", {"reportId": rid, "reportDocumentId": None, "processingStatus": state,
+ "cancelled": True,
"dataStartTime": got.get("dataStartTime"),
"dataEndTime": got.get("dataEndTime"),
"createdTime": got.get("createdTime"), "compression": None,
@@ -3419,7 +3457,7 @@ def report_document(fields, report_type, start=None, end=None, options=None,
raise AmazonError(f"the {report_type} report document carried no download URL")
text = _download_document(url, str(meta.get("compressionAlgorithm") or ""), spool=True)
say(f" {report_type}: {len(text):,} chars")
- return text, {"reportId": rid, "reportDocumentId": doc_id,
+ return text, {"reportId": rid, "reportDocumentId": doc_id, "processingStatus": state,
"dataStartTime": got.get("dataStartTime"),
"dataEndTime": got.get("dataEndTime"), "createdTime": got.get("createdTime"),
"compression": meta.get("compressionAlgorithm"), "marketplaceIds": got.get("marketplaceIds")}
diff --git a/platform/harness/amazon_store.py b/platform/harness/amazon_store.py
index 5dfc5e3da9f0ab4ceaa4d362823e35a05ae020f9..e7a7338bce015b1ab2bceb7e895f7a4236b4d990 100644
--- a/platform/harness/amazon_store.py
+++ b/platform/harness/amazon_store.py
@@ -463,27 +463,48 @@ def read_database_enablement(rt):
return {"available": True, "databases": {key: stored.get(key, True) for key in keys}}
-#: ⛔⛔ HOTFIX 2026-09-29 (owner): *"We should only sync daily at the fastest."* A scheduled pass
-#: refreshes each table at most once a day. The two hours of grace keep a table stamped late in
-#: yesterday's pass from sliding to 48 hours because today's pass reached it a little earlier.
-#: An operator's Sync now keeps only Amazon's own report floors (owner ruling, same day).
+# Day length for per-table cadence. Manual sync keeps the report's own refresh floor.
SCHEDULED_FLOOR_SECONDS = 86400
-SCHEDULED_FLOOR_GRACE_SECONDS = 2 * 3600
+SCHEDULED_FLOOR_GRACE_SECONDS = 0
+
+
+def _cadence_seconds(table, state=None):
+ """Return the table's scheduled interval, using calendar months for monthly reports."""
+ cadence = _client().CADENCE[table]
+ if cadence["period"] == "week":
+ if state:
+ stamp = _utc(state["updated_at"])
+ return int((_week_start(stamp) + timedelta(days=7) - stamp).total_seconds())
+ return 7 * SCHEDULED_FLOOR_SECONDS
+ if cadence["period"] == "month":
+ stamp = _utc(state["updated_at"]) if state else None
+ if stamp is None:
+ return 30 * SCHEDULED_FLOOR_SECONDS
+ year, month = stamp.year + (stamp.month == 12), stamp.month % 12 + 1
+ due = stamp.replace(year=year, month=month, day=1,
+ hour=0, minute=0, second=0, microsecond=0)
+ return int((due - stamp).total_seconds())
+ return SCHEDULED_FLOOR_SECONDS // cadence["perDay"]
+
+
+def sync_schedule(states, now=None):
+ """Return every SP-API table's last sync, next sync and configured passes per day."""
+ now = _utc(now)
+ schedule = []
+ for table in TABLES:
+ state = states.get(table)
+ seconds = _cadence_seconds(table, state)
+ _wait, due, _seconds, _scheduled = _interval_wait(table, state, now, floor_s=seconds)
+ if due is None and state and state.get("state") == "done":
+ vendor = _client().REPORT_TYPES[table].get("min_interval_s", 0)
+ due = _utc(state["updated_at"]) + timedelta(seconds=max(vendor, seconds))
+ schedule.append({"table": table, "lastSyncAt": state.get("updated_at") if state else None,
+ "nextSyncAt": _iso(due or now), "perDay": _client().CADENCE[table]["perDay"]})
+ return schedule
def sync_plan(states, tables=None, *, now=None, slot_pass=False):
- """Pure cadence decision: EVERY due table, oldest first, with no stagger sleep.
-
- `slot_pass` is the tenant's daily slot pass: it IS the once-a-day sync, so only Amazon's own
- report floors defer a table. Any other scheduled pass keeps the daily floor.
-
- Attempt markers survive in the tenant's saved mirror. Failed tables rotate behind peers.
- Settlement LIST discovery runs separately on every resync and consumes no report slot.
- The client still owns pacing/retries within a table.
- ⛔ HOTFIX 2026-09-29: this picked ONE table per tick. At a once-a-day cadence that is one
- table a day, about seven weeks between two refreshes of the same table, so a scheduled pass
- now takes every table whose daily floor (and Amazon's own floor) has passed.
- """
+ """Plan every due table, oldest first, with the same per-table cadence inside and outside the daily slot."""
from harness.amazon_client import REPORT_STAGGER_SECONDS
now = _utc(now)
want = list(TABLES if tables is None else tables)
@@ -491,10 +512,8 @@ def sync_plan(states, tables=None, *, now=None, slot_pass=False):
if state and table != "settlements"}
deferred = {}
due = []
- floor = 0 if slot_pass else SCHEDULED_FLOOR_SECONDS - SCHEDULED_FLOOR_GRACE_SECONDS
for table in want:
- if table == "settlements":
- continue
+ floor = _cadence_seconds(table, states.get(table))
if (reason := _interval_reason(table, states.get(table), now, floor_s=floor)):
deferred[table] = reason
else:
@@ -508,8 +527,37 @@ def sync_plan(states, tables=None, *, now=None, slot_pass=False):
for table in due:
deferred[table] = ("table stagger: waiting for a later cadence tick; "
f"minimum spacing {REPORT_STAGGER_SECONDS:g}s")
- return {"tables": selected + (["settlements"] if "settlements" in want else []),
- "deferred": deferred}
+ return {"tables": selected, "deferred": deferred}
+
+
+def scheduled_sync_plan(tenant_key, *, rt, now=None):
+ """Combine enabled tables' due plans from each connection's own resident cadence ticks."""
+ path = _tenant_mirror_path(tenant_key, rt)
+ if path.resolve() != Path(datastore.DB_PATH).resolve():
+ raise RuntimeError("Amazon scheduling requires the tenant's resident mirror")
+ enabled = read_database_enablement(rt)
+ tables = [table for table in TABLES
+ if enabled["databases"].get(amazon_schema.DATABASE_PREFIX + table)]
+ if not tables:
+ return {"tables": [], "deferred": {}}
+ now = _utc(now)
+ con, _path = _connect_tenant_mirror(tenant_key, rt)
+ try:
+ due, deferred = {}, {}
+ for entry in account_entries(tenant_key, rt):
+ prefix = _account_prefix(entry["connection_id"]) + entity_key("sync_tick_")
+ ticks = {entity[len(prefix):]: {"state": phase, "rows": rows, "updated_at": at, "detail": detail}
+ for entity, phase, rows, at, detail in con.execute(
+ "SELECT entity, phase, rows, updated_at, cursor_wd FROM _sync_state "
+ "WHERE starts_with(entity, ?)", [prefix]).fetchall()}
+ plan = sync_plan(ticks, tables, now=now)
+ due.update(dict.fromkeys(plan["tables"]))
+ for table, reason in plan["deferred"].items():
+ deferred.setdefault(table, []).append(reason)
+ return {"tables": list(due), "deferred": {
+ table: "; ".join(dict.fromkeys(reasons)) for table, reasons in deferred.items() if table not in due}}
+ finally:
+ con.close()
def _readable_duration(seconds):
@@ -525,10 +573,12 @@ def _readable_duration(seconds):
def _interval_wait(table, state, now, floor_s=None):
- """Seconds until `table` may be requested again (0 when due) and which floor decides it.
-
- `floor_s` is the scheduled daily floor; Amazon's own `min_interval_s` always applies too.
- """
+ """Return the wait and deciding refresh floor, preserving the vendor floor on manual sync."""
+ if state and state.get("state") == "throttled" and state.get("detail"):
+ due = _utc(state["detail"])
+ wait = (due - _utc(now)).total_seconds()
+ if wait > 0:
+ return wait, due, int(wait), False
vendor = (_client().REPORT_TYPES.get(table) or {}).get("min_interval_s") or 0
seconds = max(int(vendor), int(floor_s or 0))
if seconds and state and state["state"] == "done":
@@ -540,11 +590,13 @@ def _interval_wait(table, state, now, floor_s=None):
def _interval_reason(table, state, now, floor_s=None):
- wait, due, seconds, daily = _interval_wait(table, state, now, floor_s=floor_s)
+ wait, due, seconds, scheduled = _interval_wait(table, state, now, floor_s=floor_s)
if wait <= 0:
return None
- if daily:
- return (f"Scheduled syncs refresh each Amazon table at most once a day; "
+ if state and state.get("state") == "throttled":
+ return f"Amazon is limiting requests; next retry {_iso(due)}"
+ if scheduled:
+ return (f"This Amazon table refreshes every {_readable_duration(seconds)}; "
f"next due {_iso(due)}")
return (f"Amazon allows this report at most once every {_readable_duration(seconds)}; "
f"next due {_iso(due)}")
@@ -3964,7 +4016,7 @@ def _restored_state_rows(rows, tables, refused):
return rows
refused = set(refused)
names = sorted(set(tables) | {table_name(t) for t in TABLES}, key=lambda name: (-len(name), name))
- families = ("", WATERMARK_PREFIX, BACKFILL_FLOOR_PREFIX, "sync_tick_", "reconcile_", "limit_", "refused_",
+ families = ("", WATERMARK_PREFIX, BACKFILL_FLOOR_PREFIX, "sync_tick_", "resume_", "reconcile_", "limit_", "refused_",
RESYNC_PREFIX)
shared = {entity_key(key) for key in (FIRST_LOAD_END_KEY, FIRST_LOAD_STATUS_KEY, FIRST_LOAD_DUE_KEY)}
@@ -4735,7 +4787,7 @@ def _record_state(con, key, phase, rows=0, detail=None):
con.execute(
"INSERT OR REPLACE INTO _sync_state "
"(entity, phase, last_id, cursor_wd, rows, updated_at) VALUES (?,?,?,?,?,?)",
- [str(key), str(phase), None, (str(detail) if payload_key.startswith(("amzn_reconcile_", "amzn_limit_", "amzn_refused_", "amzn_pending_report_",
+ [str(key), str(phase), None, (str(detail) if payload_key.startswith(("amzn_reconcile_", "amzn_limit_", "amzn_refused_", "amzn_pending_report_", "amzn_resume_",
"amzn_resync_", "amzn_waiting_"))
else str(detail)[:300]) if detail else None, int(rows or 0),
_iso(datetime.now(timezone.utc))])
@@ -4807,31 +4859,25 @@ def _first_load_due(con):
def _restore_first_load_wait(tenant_key, con):
+ """Keep the history floor in the producer and release legacy connector-wide desk holds."""
from core import connector_sync
due = _first_load_due(con)
- if _account_scope() is not None:
- return due
- connector_sync.defer_sync("amazon_sp", tenant_key, due.timestamp() if due else None)
+ connector_sync.clear_sync_defer("amazon_sp", tenant_key)
return due
def _loaded_refresh_hold(con, table, now, *, slot_pass=False):
- """`(next due, reason)` while a loaded table's own floor holds it during a first load, else None.
-
- A weekly table is due when a new week has closed after its watermark (D-11328). Every other
- table, settlement discovery included, keeps the daily floor and Amazon's own floor.
- """
+ """Return a loaded table's cadence hold while its first load continues, or None when due."""
now = _utc(now)
tick = _read_state(con, entity_key("sync_tick_" + table))
# A queued resync is a repair: only Amazon's own floor holds it.
repair = queued_resync(con, table) is not None
- floor = 0 if slot_pass or repair else SCHEDULED_FLOOR_SECONDS - SCHEDULED_FLOOR_GRACE_SECONDS
- wait, due, _seconds, daily = _interval_wait(table, tick, now, floor_s=floor)
+ floor = 0 if repair else _cadence_seconds(table, tick)
+ wait, due, _seconds, _scheduled = _interval_wait(table, tick, now, floor_s=floor)
reason = None
if wait > 0:
# The card parses "next due " out of a reason, so every hold keeps that phrase.
- reason = (f"Each loaded Amazon table refreshes at most once a day while the first load continues. "
- f"It is next due {_iso(due)}" if daily else _interval_reason(table, tick, now, floor_s=floor))
+ reason = _interval_reason(table, tick, now, floor_s=floor)
mark = read_watermark(con, table)
if _client().REPORT_TYPES.get(table, {}).get("weekly") and mark and not repair:
week = _week_start(mark) + timedelta(days=7)
@@ -5129,6 +5175,8 @@ def first_load_status(tenant_key, rt=None):
remaining = int(values["remaining"])
# S82-T14: one pass sees only its own tables. The job's store record counts the whole load.
job = first_load_job_summary(rt) if _store_documents(rt) else None
+ if job:
+ job["tables"]["inventory"] = inventory_table_status(con)
if job and isinstance(job.get("remaining"), int) and state["state"] != "done":
remaining = job["remaining"]
out = {"state": state["state"], **first_load_payload(markers, remaining,
@@ -5413,7 +5461,7 @@ def _sync_writer(tenant_key, rt, log):
def _sync_delegate(tenant_key, rt, log):
"""Every table but the job's own takes one bounded first-load pass: settlements, period, Group A, SQP.
- The same pass refreshes each loaded table whose daily floor is due, so loaded data keeps moving.
+ The same pass refreshes each loaded table whose cadence is due, so loaded data keeps moving.
"""
def delegate(exclude, budget):
tables = None if exclude is None else [t for t in TABLES if t not in set(exclude)]
@@ -5473,7 +5521,91 @@ def _status_stamp(status, connector):
return max(stamps, key=lambda item: item[0])[1] if stamps else None
-def first_load_table_status(tasks, finished, failed, counts):
+def inventory_table_status(con):
+ """Read inventory counts and account-scoped source proof without changing the mirror or requesting reports."""
+ rows = (int(con.execute("SELECT COUNT(*) FROM amzn_inventory").fetchone()[0])
+ if _columns_of(con, "amzn_inventory") else 0)
+ out = {"rows": rows, "empty": rows == 0}
+ if rows:
+ return out
+ keys = ("amzn_inventory", "amzn_reconcile_inventory", "amzn_refused_inventory", "amzn_inventory_awd")
+ clauses = " OR ".join("entity=? OR ends_with(entity, ?)" for _ in keys)
+ saved = con.execute(
+ "SELECT entity, phase, cursor_wd, rows, updated_at FROM _sync_state WHERE " + clauses
+ + " OR contains(entity, 'amzn_pending_report_')",
+ [value for key in keys for value in (key, "__" + key)]).fetchall()
+ accounts, pending, unknown_request = {}, [], False
+ for entity, phase, detail, count, stamp in saved:
+ prefix, key = (str(entity).split("__", 1) if str(entity).startswith("amzn_account_")
+ and "__" in str(entity) else ("", str(entity)))
+ if key.startswith("amzn_pending_report_"):
+ try:
+ request = json.loads(detail or "{}")
+ except (ValueError, TypeError):
+ request = {}
+ if not isinstance(request, dict):
+ request = {}
+ if request.get("table") == "inventory":
+ pending.append({"type": request.get("reportType"), "state": phase,
+ "reportId": request.get("reportId"),
+ "processingStatus": request.get("processingStatus"), "updatedAt": stamp})
+ elif not request.get("table"):
+ unknown_request = True
+ continue
+ accounts.setdefault(prefix, {})[key] = {"state": phase, "detail": detail,
+ "rows": count, "updated_at": stamp}
+ if pending:
+ return {**out, "code": "first_load_pending", "reason":
+ "Amazon is preparing the inventory report, so keep sync enabled and retry after it completes.",
+ "reports": pending}
+ # Account markers own their proof once account scoping exists.
+ scoped = {key: state for key, state in accounts.items() if key}
+ accounts = scoped or accounts
+ reports = []
+ for states in accounts.values():
+ marker = states.get("amzn_inventory") or {}
+ refused = refused_reports(states)
+ if refused:
+ return {**out, "code": "refused_by_amazon", "reason": refused[0]["reason"]}
+ proof = states.get("amzn_reconcile_inventory") or {}
+ try:
+ detail = json.loads(proof.get("detail") or "{}")
+ except (ValueError, TypeError):
+ detail = {}
+ if not isinstance(detail, dict):
+ detail = {}
+ dates = detail.get("dates")
+ valid_dates = (isinstance(dates, dict) and bool(dates)
+ and all(isinstance(day, str) and re.fullmatch(r"\d{4}-\d{2}-\d{2}", day)
+ and type(count) is int and count == 0 for day, count in dates.items()))
+ awd = states.get("amzn_inventory_awd") or {}
+ if (marker.get("state") != "done" or marker.get("rows") != 0 or proof.get("state") != "done"
+ or type(detail.get("count")) is not int or detail["count"] != 0
+ or not isinstance(detail.get("document"), str) or not detail["document"].strip()
+ or detail.get("column") != "_snapshot_date" or not valid_dates
+ or detail.get("start") != min(dates) or detail.get("end") != max(dates)
+ or awd.get("state") != "done" or awd.get("rows") != 0):
+ return {**out, "code": "first_load_pending", "reason":
+ "Inventory source checks are incomplete, so retry Amazon sync to finish the FBA and AWD inventory checks."}
+ reports.append({"type": _client().REPORT_TYPES["inventory"]["report"], "state": proof["state"],
+ "reportId": detail.get("reportId"), "processingStatus": detail.get("processingStatus"),
+ "documentId": detail["document"], "start": detail["start"], "end": detail["end"],
+ "rows": detail["count"], "updatedAt": proof.get("updated_at")})
+ if not reports:
+ if unknown_request:
+ return {**out, "code": "first_load_pending", "reason":
+ "A saved Amazon request has no table identity, so retry sync to refresh the inventory report status."}
+ return {**out, "code": "not_requested", "reason":
+ "No inventory report request is saved, so start Amazon sync to request the FBA ledger."}
+ periods = {(report["start"], report["end"]) for report in reports}
+ start, end = next(iter(periods))
+ period = (start if start == end else f"{start} to {end}") if len(periods) == 1 else "the loaded dates"
+ return {**out, "code": "no_source_rows", "reason":
+ f"Amazon returned no FBA ledger rows for {period} and no AWD inventory, so check seller stock and retry Amazon sync.",
+ "reports": reports}
+
+
+def first_load_table_status(tasks, finished, failed, counts, *, table_status=None):
"""D-2643, D-2642, D-2649: a row count or a named reason per table, and why orders are behind."""
out = {}
for table in dict.fromkeys(EMPTY_REASON_TABLES + BEHIND_TABLES):
@@ -5487,17 +5619,25 @@ def first_load_table_status(tasks, finished, failed, counts):
f"The first load is still loading this table. {len(left)} report windows remain."
if left else "Amazon returned no rows for this seller in the loaded period."
if own else "This table holds no rows and the first load owes it no windows.")
+ if table == "inventory":
+ row.update(empty=True, code="first_load_pending", reason=
+ "Inventory source proof is not available in this job record, so retry Amazon sync to refresh its status.")
if table in BEHIND_TABLES and left:
done = [t for t in own if t["id"] in finished or t.get("landed")]
oldest = min((t["start"] for t in done if t.get("start")), default=None)
row["behind"] = (f"The first load is still loading this table. {len(left)} of {len(own)} report "
"windows remain" + (f", and history before {oldest[:10]} is not loaded yet." if oldest
else ", starting with the most recent month."))
+ if table_status and table in table_status:
+ row.update(table_status[table])
+ if not row.get("empty"):
+ row.pop("reason", None)
+ row.pop("code", None)
out[table] = row
return out
-def first_load_job_summary(docs):
+def first_load_job_summary(docs, *, table_status=None):
"""The latest generation record and the cursor's remaining count, read from the store."""
try:
record = _doc(docs, FIRST_LOAD_GENERATIONS_KEY)
@@ -5511,7 +5651,7 @@ def first_load_job_summary(docs):
"remaining": record.get("remaining", last.get("remainingAfter")),
"tablesPublished": list(last.get("tablesPublished") or []), "end": last.get("end"),
"operation": last.get("operation"), "resume_at": last.get("resume_at"),
- "tables": last.get("tables") or {}}
+ "tables": {**(last.get("tables") or {}), **(table_status or {})}}
def _record_generation(docs, row):
@@ -5787,7 +5927,15 @@ def run_first_load_generation(tenant_key, rt=None, *, generation=1, budget_secon
row_counts = counts(list(dict.fromkeys(EMPTY_REASON_TABLES + BEHIND_TABLES)))
except Exception: # noqa: BLE001
row_counts = {}
- record["tables"] = first_load_table_status(after, finished, failed, row_counts)
+ table_status = None
+ if plan is None:
+ con, _path = _connect_tenant_mirror(key, rt)
+ try:
+ table_status = {"inventory": inventory_table_status(con)}
+ finally:
+ con.close()
+ record["tables"] = first_load_table_status(after, finished, failed, row_counts,
+ table_status=table_status)
record["state"] = "done" if not record["remainingAfter"] else "partial"
status_after = read_connector_status(docs)
record["odooLastGoodSyncAt"] = {"start": _status_stamp(status_before, "odoo"),
@@ -5812,7 +5960,7 @@ def run_first_load_generation(tenant_key, rt=None, *, generation=1, budget_secon
pass
-def sync_window(table, mark, *, now, since=None, backfill_days=None):
+def sync_window(table, mark, *, now, since=None, backfill_days=None, intraday=False):
"""Pure request decision for one table: start, end, report, incremental and cause."""
end = _utc(now)
days = DEFAULT_BACKFILL_DAYS if backfill_days is None else backfill_days
@@ -5830,6 +5978,11 @@ def sync_window(table, mark, *, now, since=None, backfill_days=None):
start, incremental = end - timedelta(days=days), False
cause = f"first full load: {days} days of backfill"
spec = _client().REPORT_TYPES.get(table) or {}
+ if incremental and not spec.get("snapshot") and not _periodic(spec):
+ cadence = _client().CADENCE[table]
+ lookback = cadence["intradayLookbackDays" if intraday else "lookbackDays"]
+ start = min(start, end - timedelta(days=lookback))
+ cause += f"; re-read {lookback} days ({'intraday' if intraday else 'daily'})"
rtype = spec.get("report_incremental") if incremental and spec.get("report_incremental") else spec.get("report")
rtype = rtype or spec.get("api")
if table in ("economics", "order_metrics"):
@@ -5858,7 +6011,22 @@ def sync_window(table, mark, *, now, since=None, backfill_days=None):
# The watermark filters issued reports by creation time; it never requests one.
rtype = spec.get("report_discover")
return {"start": start, "end": end, "report": rtype,
- "incremental": incremental, "cause": cause}
+ "incremental": incremental, "intraday": intraday, "cause": cause}
+
+
+def _lookback_intraday(con, table, now, *, slot_pass=False):
+ """Use the daily window until this account has completed it, including after a failed attempt."""
+ day = _read_state(con, entity_key("lookback_day_" + table))
+ return bool(not slot_pass and day and day.get("state") == "done"
+ and day.get("detail") == now.date().isoformat())
+
+
+def _record_lookback_day(con, table, plan, now):
+ """Only a completed incremental daily request earns this account's intraday window."""
+ if (plan["incremental"] and not plan.get("intraday")
+ and plan["end"] >= now - timedelta(minutes=2)
+ and not _lookback_intraday(con, table, now)):
+ _record_state(con, entity_key("lookback_day_" + table), "done", detail=now.date().isoformat())
def _sync_event(kind, tenant, table, detail, rt, report, log):
@@ -5955,10 +6123,21 @@ def _pending_report_key(table, report_type, **kwargs):
return entity_key("pending_report_" + hashlib.sha256(identity.encode()).hexdigest())
+def _save_throttled_resume(con, table, exc, plan, completed, *, first_load, since):
+ """Keep the continuation beside the account's markers and leave the retry tick as an ISO deadline."""
+ continuation = {**exc.as_resume(), "plan": {**plan, "start": _iso(plan["start"]), "end": _iso(plan["end"])},
+ "completed": completed, "first_load": first_load,
+ "since": _iso(_utc(since)) if since is not None else None}
+ _record_state(con, entity_key("resume_" + table), "throttled", detail=json.dumps(continuation))
+ due = datetime.now(timezone.utc) + timedelta(seconds=max(60.0, float(exc.retry_after or 60)))
+ _record_state(con, entity_key("sync_tick_" + table), "throttled", detail=_iso(due))
+ return max(0, (due - datetime.now(timezone.utc)).total_seconds())
+
+
@contextmanager
def _report_resume(con, table, report_type, **kwargs):
"""Resume the same report/window for up to 24 hours instead of paying createReport again."""
- from harness.amazon_client import AmazonReportError, AmazonReportPending
+ from harness.amazon_client import AmazonReportError, AmazonReportPending, AmazonThrottled
window = {key: kwargs.get(key) for key in ("start", "end", "options")}
key = _pending_report_key(table, report_type, **kwargs)
state = _read_state(con, key)
@@ -5973,7 +6152,15 @@ def _report_resume(con, table, report_type, **kwargs):
yield kwargs
except AmazonReportPending as exc:
_record_state(con, key, "pending", detail=json.dumps({
- "reportId": exc.report_id, "createdAt": saved.get("createdAt", stamp), "window": window}))
+ "reportId": exc.report_id, "createdAt": saved.get("createdAt", stamp), "window": window,
+ "table": table, "reportType": report_type, "processingStatus": exc.status}))
+ raise
+ except AmazonThrottled as exc:
+ if exc.report_id is not None:
+ _record_state(con, key, "throttled", detail=json.dumps({
+ **exc.as_resume(), "reportId": str(exc.report_id),
+ "createdAt": saved.get("createdAt", stamp), "window": window,
+ "table": table, "reportType": report_type}))
raise
except AmazonReportError:
con.execute("DELETE FROM _sync_state WHERE entity=?", [key])
@@ -6012,6 +6199,21 @@ def _record_reconciliation(con, table, marker):
_record_state(con, key, "done", detail=json.dumps(marker, separators=(",", ":")))
+def _restatement_may_overlap(con, mirror, predicate, params, start, end, *, unsafe="TRUE"):
+ """Treat absent or invalid legacy period metadata as unknown when checking retained rows."""
+ columns = set(_columns_of(con, mirror))
+ def period(name):
+ return f'TRY_CAST("{name}" AS TIMESTAMPTZ)' if name in columns else "NULL::TIMESTAMPTZ"
+ old_start, old_end = period("_period_start"), period("_period_end")
+ return bool(con.execute(
+ f'SELECT 1 FROM {mirror} WHERE NOT COALESCE(({predicate}), FALSE) AND ({unsafe}) '
+ f'AND ({old_start} IS NULL OR TRY_CAST(? AS TIMESTAMPTZ) IS NULL '
+ f'OR {old_start} < TRY_CAST(? AS TIMESTAMPTZ)) '
+ f'AND ({old_end} IS NULL OR {old_start} IS NULL OR {old_end} <= {old_start} '
+ f'OR {old_end} > TRY_CAST(? AS TIMESTAMPTZ)) LIMIT 1',
+ [*params, end, end, start]).fetchone())
+
+
def _replace_period(con, table, rows, columns, column, start, end=None, marker=None, *, snapshot=False):
"""Atomic range/date replacement and proof; an empty restatement removes old rows too.
@@ -6041,10 +6243,44 @@ def _replace_period(con, table, rows, columns, column, start, end=None, marker=N
if column != "_snapshot_date" or end:
raise ValueError("a snapshot replacement is keyed by one _snapshot_date")
predicate, params = "TRUE", []
+ source_keys = _client().REPORT_TYPES[table].get("restatement_keys", ())
+ staged = None
+ if source_keys:
+ overlap_end = (marker or {}).get("end") or end
+ missing = set(source_keys) - set(columns)
+ if missing:
+ # A same-window replay remains safe without source keys; a moving overlap must refuse.
+ if _restatement_may_overlap(con, mirror, predicate, params, start, overlap_end):
+ raise ValueError(f"{table}: overlapping restatement omitted source keys: "
+ + ", ".join(sorted(missing)) + "; request the complete source report")
+ elif rows:
+ staged = "amazon_restatement_" + uuid.uuid4().hex
+ names = ", ".join(_quoted_identifier(c) for c in columns)
+ con.execute(f"CREATE TEMP TABLE {staged} AS SELECT {names} FROM {mirror} WHERE FALSE")
+ try:
+ _replace(con, staged, columns, types, rows, source_table=mirror)
+ required = [c for c in source_keys if c not in ("fnsku", "asin")]
+ invalid = " OR ".join(f'NULLIF(TRIM(CAST({_quoted_identifier(c)} AS VARCHAR)), \'\') IS NULL'
+ for c in required)
+ if con.execute(f"SELECT 1 FROM {staged} WHERE {invalid} LIMIT 1").fetchone():
+ raise ValueError(f"{table}: restatement omitted a source key value; request the complete source report")
+ if _restatement_may_overlap(con, mirror, predicate, params, start, overlap_end, unsafe=invalid):
+ raise ValueError(f"{table}: overlapping legacy rows omitted a source key value; "
+ "restore the complete source report before restating")
+ except BaseException:
+ con.execute(f"DROP TABLE {staged}")
+ raise
con.execute("BEGIN TRANSACTION")
try:
con.execute(f'DELETE FROM {mirror} WHERE {predicate}', params)
- count = _replace(con, mirror, columns, types, rows)
+ if staged:
+ matches = " AND ".join(f'{mirror}.{_quoted_identifier(c)} IS NOT DISTINCT FROM '
+ f'k.{_quoted_identifier(c)}' for c in source_keys)
+ con.execute(f'DELETE FROM {mirror} WHERE EXISTS (SELECT 1 FROM {staged} k WHERE {matches})')
+ con.execute(f"INSERT INTO {mirror} ({names}) SELECT {names} FROM {staged}")
+ count = con.execute(f"SELECT COUNT(*) FROM {staged}").fetchone()[0]
+ else:
+ count = _replace(con, mirror, columns, types, rows)
if marker is not None:
_record_reconciliation(con, table, marker)
if table == "listing_items":
@@ -6054,6 +6290,9 @@ def _replace_period(con, table, rows, columns, column, start, end=None, marker=N
except Exception:
con.execute("ROLLBACK")
raise
+ finally:
+ if staged:
+ con.execute(f"DROP TABLE {staged}")
def document_row_count(text, roots=()):
@@ -6196,6 +6435,7 @@ def _replace_legacy_snapshot(con, table, columns, rows, meta, source_count):
dates = sorted(counts)
marker = {"start": dates[0], "end": dates[-1], "column": "_snapshot_date",
"count": source_count, "dates": counts, "document": meta.get("reportDocumentId")}
+ marker.update({key: meta[key] for key in ("reportId", "processingStatus") if meta.get(key)})
# Every earlier date and any undated pre-migration copy goes; AWD rows are another leg.
predicate = "COALESCE(inventory_source, 'FBA') <> 'AWD'" if table == "inventory" else "TRUE"
con.execute("BEGIN TRANSACTION")
@@ -7470,7 +7710,8 @@ def _sync_accounts_candidate(tenant_key="royal-imports", tables=None, since=None
for entry in entries:
cid = entry["connection_id"]
try:
- with _account_workspace(con, tenant_key, rt, entry) as scope:
+ with _account_workspace(con, tenant_key, rt, entry) as scope, \
+ _client().rate_limit_scope(tenant_key, cid):
_sync_credential(tenant_key, rt, connection_id=cid)
result = _sync_candidate(tenant_key, tables=tables, since=since, log=log, rt=rt,
max_wait=max_wait, scheduled=scheduled, first_load=first_load,
@@ -7641,7 +7882,7 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
ticks.update({name: None for name in (table, pairs.get(table)) if name in want})
for table in want:
tick = ticks.get(table)
- # A queued resync is a repair: only Amazon's own floor and the stagger hold it, never the daily floor.
+ # A queued resync is a repair: only Amazon's own floor and the stagger hold it.
if (tick and tick["state"] == "done" and queued_resync(con, table) is not None
and read_watermark(con, table) and not _interval_reason(table, tick, end)):
ticks[table] = {**tick, "state": "resync"}
@@ -7775,6 +8016,15 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
else:
for table in pending:
mark = read_watermark(con, table)
+ if table == "settlements" and since is None:
+ # Empty discovery earns a request cutoff, never a stored source watermark.
+ tick = _read_state(con, entity_key("sync_tick_" + table))
+ if tick and tick.get("state") == "done":
+ mark = _utc(tick["updated_at"])
+ elif not mark:
+ day = _read_state(con, entity_key("lookback_day_" + table))
+ if day and day.get("state") == "done":
+ mark = _utc(day["updated_at"])
if not mark and since is None and table != "settlements" and table in completed:
# A local fallback cannot reopen old daily history or invent a stored watermark.
recent = end - timedelta(days=FIRST_LOAD_RECENT_DAYS)
@@ -7784,7 +8034,8 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
mark = recent
if not recent <= mark <= end:
mark = recent
- plan = plans[table] = sync_window(table, mark, now=end, since=since)
+ plan = plans[table] = sync_window(table, mark, now=end, since=since,
+ intraday=_lookback_intraday(con, table, end, slot_pass=slot_pass))
if table == SQP_TABLE:
months = _month_windows(plan["start"], plan["end"])
if not plan["incremental"]:
@@ -7798,6 +8049,16 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
else:
work.append((table, None, None, None))
+ resumes = {}
+ for table in pending:
+ saved = _read_state(con, entity_key("resume_" + table))
+ continuation = json.loads(saved["detail"]) if saved and saved.get("detail") else {}
+ if (continuation and continuation.get("first_load") == first_load
+ and continuation.get("since") == (_iso(_utc(since)) if since is not None else None)):
+ resumes[table] = continuation
+ plans[table] = {**continuation["plan"], "start": _utc(continuation["plan"]["start"]),
+ "end": _utc(continuation["plan"]["end"])}
+
if sqp_limit is not None:
def continuation_first(task):
_table, w0, w1, batch = task
@@ -7834,6 +8095,14 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
and (task_end is not None or table == "settlements"
or table in PERIOD_TABLES or table in LEGACY_SNAPSHOT_TABLES)):
break
+ tick = _read_state(con, entity_key("sync_tick_" + table))
+ if first_load and (tick or {}).get("state") == "throttled":
+ reason = _interval_reason(table, tick, datetime.now(timezone.utc))
+ if reason:
+ report["tables"][table] = {"state": "deferred", "reason": reason}
+ wait = _interval_wait(table, tick, datetime.now(timezone.utc))[0]
+ report["nextDueSeconds"] = min(report.get("nextDueSeconds") or wait, wait)
+ continue
if table == SQP_TABLE and task_start is not None and _sqp_period(task_start, task_end) == "WEEK":
weekly_limit = {**sqp_limit, "refusedAsins": {}, "errors": []}
causes = []
@@ -7856,6 +8125,16 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
report["nextDueSeconds"] = min(report.get("nextDueSeconds") or 900, 900)
_record_state(con, entity_key(table), "pending", 0, weekly_pending["reason"])
except Exception as exc:
+ if isinstance(exc, cx.AmazonThrottled):
+ problem = _sync_problem(exc, table, exc.operation or "report_document", tenant_key, rt, report, log)
+ report["tables"][table] = problem
+ failed.add(table)
+ pending_reports.add(table)
+ wait = _save_throttled_resume(con, table, exc, plans[table], [],
+ first_load=first_load, since=since)
+ report["nextDueSeconds"] = min(report.get("nextDueSeconds") or wait, wait)
+ _record_state(con, entity_key(table), "throttled", detail=problem["reason"])
+ continue
causes.append(str(exc))
if isinstance(exc, cx.AmazonError) and not isinstance(exc, (cx.AmazonReportPending, cx.AmazonThrottled)):
marker = _sqp_month(con, task_start, task_end)
@@ -7872,7 +8151,6 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
continue
if first_load and (table in GROUP_A_TABLES or (table in prior_refusals
and cx.REPORT_TYPES[table].get("min_interval_s"))):
- tick = _read_state(con, entity_key("sync_tick_" + table))
reason = _interval_reason(table, tick, datetime.now(timezone.utc))
if reason:
report["tables"][table] = {"state": "deferred", "reason": reason}
@@ -7932,6 +8210,7 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
proof_key = entity_key("reconcile_" + table)
previous_proof = _read_state(con, proof_key) if table in invalid_proofs else None
resync, resync_count = [], 0
+ completed_windows = list(resumes.get(table, {}).get("completed", []))
try:
if previous_proof:
# The recorder accepts an empty object; only a parsed document can replace it with proof.
@@ -7941,9 +8220,9 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
_record_state(con, entity_key("sync_tick_" + table), "attempted")
if sibling and not first_load:
_record_state(con, entity_key("sync_tick_" + sibling), "attempted")
- sibling_written, sibling_cleared = 0, False
- written, cols, cleared = 0, [], False
- landed_window = False
+ sibling_written, sibling_cleared = 0, bool(completed_windows)
+ written, cols, cleared = 0, [], bool(completed_windows)
+ landed_window = bool(completed_windows)
pricing_asins = []
operations = {}
if first_load:
@@ -7993,12 +8272,17 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
"state": "done", "reports": len(discovered),
"downloaded": downloaded, "skipped": skipped}
if not discovered:
+ con.execute("DELETE FROM _sync_state WHERE entity=?", [entity_key("resume_" + table)])
report["tables"][table] = {"rows": 0, "operations": operations,
"state": "empty", "reason": "No issued settlement reports found."}
landed.add((table, task_start, task_end, task_batch))
if first_load:
_record_state(con, key, "done", 0, "No issued settlement reports found.")
- # Empty discovery earns no cursor, attempt stamp or document marker.
+ # A successful empty scheduled discovery earns cadence, but no source cursor or document marker.
+ if scheduled and not first_load:
+ _record_state(con, entity_key("sync_tick_" + table), "done", 0)
+ if not first_load and since is None:
+ _record_lookback_day(con, table, {**plan, "incremental": True}, end)
continue
if first_load and settlement_remaining:
break
@@ -8047,7 +8331,8 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
plan["cause"] = (f"resync of {len(margin_windows)} stored windows "
f"from {_iso(margin_windows[0][0])}")
log(f"{table}: {plan['cause']}")
- elif table in GROUP_A_TABLES and not first_load and len(margin_windows) > 1:
+ elif (table in GROUP_A_TABLES and not first_load and len(margin_windows) > 1
+ and not spec.get("restatement_keys")):
# One due window per table tick respects the vendor floor. The watermark
# advances only to that window's end, so the following tick loses no history.
remaining_windows = len(margin_windows) - 1
@@ -8063,6 +8348,9 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
if not reason.startswith("Amazon keeps this report back to "):
_record_state(con, entity_key("limit_" + table), "done", detail="{}")
for w0, w1 in margin_windows:
+ window = [_iso(w0) if w0 is not None else None, _iso(w1) if w1 is not None else None]
+ if window in completed_windows:
+ continue
# One detail window is one budget task, including its reconciliation query.
probe = _DocumentProbe(table, spec.get("report") or operation)
documents += 1
@@ -8078,6 +8366,7 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
spend_rows(count)
written += count
landed_window = True
+ completed_windows.append(window)
if table in LEGACY_SNAPSHOT_TABLES:
# The dated replacement already landed; AWD must not clear that
# document or historical FBA/AWD rows on an explicit first load.
@@ -8101,6 +8390,9 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
plan["end"] = windows[0][1]
weekly = bool(table != "settlements" and cx.REPORT_TYPES[table].get("weekly"))
for w0, w1 in (() if table == "settlements" else windows):
+ window = [_iso(w0), _iso(w1)]
+ if window in completed_windows:
+ continue
# W59-T43 / C2 has no amazon.access kind. Record each credential-backed
# vendor attempt as a sync ACCESS phase, distinct from the completion event.
# Lane F/combine own the vocabulary decision; no credential or row enters C2.
@@ -8179,6 +8471,7 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
con.execute(f"DELETE FROM {table_name(empty_table)} WHERE 1=1")
cleared = True
else:
+ completed_windows.append(window)
continue
if sibling:
sibling_mirror = table_name(sibling)
@@ -8219,6 +8512,7 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
_clear_week_wait(con, table, w0)
probe.finish(len(rows) if cols else 0)
spend_rows(len(rows) if cols else 0)
+ completed_windows.append(window)
if first_load:
for reached in (table, sibling):
if reached:
@@ -8268,6 +8562,8 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
log(f" pricing: getCompetitiveSummary {response_rows:,} ASIN responses, "
f"{updated:,} mirror rows enriched")
except Exception as e: # noqa: BLE001
+ if isinstance(e, cx.AmazonThrottled):
+ raise
problem = _sync_problem(e, table, "getCompetitiveSummary", tenant_key,
rt, report, log)
_record_state(con, state_key, problem["state"], 0, problem["reason"])
@@ -8307,6 +8603,7 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
report["tables"][table]["nextWeek"] = _iso(period_repair_next)
_record_state(con, key, "done", total_rows)
_record_state(con, entity_key("sync_tick_" + table), "done", total_rows)
+ con.execute("DELETE FROM _sync_state WHERE entity=?", [entity_key("resume_" + table)])
if weekly and landed_window:
# A landed week closes any earlier refusal of this table.
con.execute("DELETE FROM _sync_state WHERE entity=?", [entity_key("refused_" + table)])
@@ -8322,6 +8619,10 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
or landed_window or (first_load and floors[table] is not None))
if mark is not None and has_window:
record_watermark(con, table, mark, total_rows)
+ if not first_load and not resync:
+ completed_plan = ({**plan, "incremental": True}
+ if table == "settlements" and since is None else plan)
+ _record_lookback_day(con, table, completed_plan, end)
_sync_event("amazon.sync", tenant_key, table,
{"state": "done", "rows": total_rows,
"operations_ok": all(v["state"] == "done" for v in operations.values())},
@@ -8341,7 +8642,10 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
"incremental": sibling_plan["incremental"], "operations": {}}
_record_state(con, entity_key(sibling), "done", sibling_total)
_record_state(con, entity_key("sync_tick_" + sibling), "done", sibling_total)
+ con.execute("DELETE FROM _sync_state WHERE entity=?", [entity_key("resume_" + sibling)])
record_watermark(con, sibling, sibling_plan["end"], sibling_total)
+ if not first_load:
+ _record_lookback_day(con, sibling, sibling_plan, end)
_sync_event("amazon.sync", tenant_key, sibling,
{"state": "done", "rows": sibling_total, "operations_ok": True},
rt, report, log)
@@ -8363,9 +8667,17 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
# A failed pass retains its previous watermark so its window will be retried.
# ⚠ ONE TABLE'S REFUSAL IS NOT THE SYNC'S. A 403 on brand_analytics is R5's UNPROVEN
# ticket answering itself, and it must not cost the six tables that did work.
- problem = _sync_problem(e, table, operation, tenant_key, rt, report, log)
+ problem = _sync_problem(e, table, getattr(e, "operation", None) or operation, tenant_key, rt, report, log)
failed.add(table)
report["tables"][table] = {"rows": 0, **problem}
+ if problem["state"] == "throttled":
+ landed.discard((table, task_start, task_end, task_batch))
+ pending_reports.add(table)
+ if _columns_of(con, mirror):
+ report["tables"][table]["rows"] = con.execute(f"SELECT COUNT(*) FROM {mirror}").fetchone()[0]
+ wait = _save_throttled_resume(con, table, e, plan, completed_windows,
+ first_load=first_load, since=since)
+ report["nextDueSeconds"] = min(report.get("nextDueSeconds") or wait, wait)
if problem["state"] == "pending":
pending_reports.add(table)
report["nextDueSeconds"] = min(report.get("nextDueSeconds") or 900, 900)
@@ -8383,8 +8695,13 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
"reason": problem["reason"][:500], "amazonStatus": problem["amazonStatus"],
"diagnostic": problem.get("diagnostic", "")[:500],
"error": problem.get("error", ""), "attempts": attempts}))
- _record_state(con, entity_key("sync_tick_" + refused),
- "attempted" if problem["state"] == "pending" else "done", 0)
+ if problem["state"] == "throttled":
+ if refused != table:
+ _save_throttled_resume(con, refused, e, sibling_plan, completed_windows,
+ first_load=first_load, since=since)
+ else:
+ _record_state(con, entity_key("sync_tick_" + refused),
+ "attempted" if problem["state"] == "pending" else "done")
except Exception: # noqa: BLE001
pass
_sync_event("amazon.sync", tenant_key, table,
@@ -10612,8 +10929,8 @@ def group_a_selftest(log=print, check=None):
clock[0] += timedelta(seconds=cx.REPORT_TYPES["long_term_storage_fees"]["min_interval_s"] + 60)
idle = sync(rt.key, tables=("long_term_storage_fees",), rt=rt, log=lambda *_: None,
scheduled=True)["tables"]["long_term_storage_fees"]
- ck("NC: with the queue empty the daily floor defers the table again",
- idle.get("state") == "deferred" and "once a day" in idle.get("reason", ""))
+ ck("NC: with the queue empty the table cadence defers it again",
+ idle.get("state") == "deferred" and "next due " in idle.get("reason", ""))
finally:
con.close()
diff --git a/platform/modules/account_health.py b/platform/modules/account_health.py
index 7df140ce5a08b6a924fd4103a7ff3b0aaac46094..e6ab99ff75deecbf6ca8550eb8fb0a2151cd0217 100644
--- a/platform/modules/account_health.py
+++ b/platform/modules/account_health.py
@@ -24,6 +24,7 @@ import hashlib
import json
import math
from pathlib import Path
+from urllib.parse import urlencode
APP_LABEL = "Account health"
SOURCE_KIND = "account_health"
@@ -55,6 +56,11 @@ SETTLEMENT_TABLE = "amzn_settlements"
COMPLAINT_METRICS = ("received_intellectual_property_complaints", "suspected_intellectual_property_violations",
"product_authenticity_customer_complaints")
SOURCE = "ut_amzn_account_health"
+LISTING_SOURCE = "ut_amzn_listing_items"
+CATEGORY_SOURCE_NOTE = (
+ "Amazon's account health report provides category totals, targets and reporting windows, but no violation records. "
+ "Related listings show listing issues. Amazon does not link them to this category."
+)
NOTIFICATION_KIND = "account_health_new_items"
_sync_delivery = None
@@ -463,7 +469,7 @@ def _page_bounds(offset, limit, total):
return start, size
-def summary_page(tenant_key, *, search="", offset=0, limit=PAGE_SIZE, metric=None, st=None):
+def summary_page(tenant_key, *, search="", offset=0, limit=PAGE_SIZE, metric=None, category=None, st=None):
"""Read one metric page or one exact metric without loading the complete snapshot."""
st = _rt(tenant_key, st)
distance = settings(tenant_key, st=st)["alertDistancePct"]
@@ -495,6 +501,8 @@ def summary_page(tenant_key, *, search="", offset=0, limit=PAGE_SIZE, metric=Non
"ELSE CASE WHEN CONTAINS(UPPER(condition), 'EQUAL') OR target = 0 THEN 'At most' ELSE 'Below' END END) "
"|| ' ' || CASE WHEN unit = 'percent' THEN printf('%.2f%%', target) ELSE CAST(target AS VARCHAR) END END, "
"condition, window_from, window_to)), ?)", [str(search or "").strip().lower()])
+ if category is not None:
+ where, filters = f"({where}) AND key = ?", [*filters, str(category)]
folded = ", ".join(f"'{s}'" for s in FOLDED)
total, folded_count = cur.execute(
sql + f" SELECT COUNT(*), COUNT(*) FILTER (WHERE status IN ({folded})) FROM metrics WHERE {where}",
@@ -530,6 +538,110 @@ def summary_page(tenant_key, *, search="", offset=0, limit=PAGE_SIZE, metric=Non
"cursor": str(start + size) if start + size < out["metricTotal"] else None}
+def _listing_array(value):
+ """Decode a connector JSON array without inventing missing entries."""
+ if isinstance(value, str):
+ try:
+ value = json.loads(value)
+ except (TypeError, ValueError):
+ return None
+ return value if isinstance(value, list) else None
+
+
+def _related_listings(tenant_key, st, user, market):
+ """Read all permitted listing issue pages through the listing source's own wall."""
+ from core import perm_scope
+ st = _rt(tenant_key, st)
+ if user is None:
+ return [], "Related listings need access to the listing source."
+ related, cursor, incomplete = [], None, False
+ try:
+ while True:
+ page = perm_scope.scoped_table_page(user, LISTING_SOURCE, st=st, cursor=cursor, limit=1024)
+ if "issues" not in {field.get("key") for field in page["fields"]}:
+ return [], "Listing issues are unavailable with your current source access."
+ for row in page["rows"]:
+ if market and row.get("marketplace_id") and str(row["marketplace_id"]) != market:
+ continue
+ issues = _listing_array(row.get("issues"))
+ if issues is None:
+ incomplete = True
+ continue
+ summaries = _listing_array(row.get("summaries")) or []
+ summaries = [item for item in summaries if isinstance(item, dict)]
+ matching = [item for item in summaries if not market or item.get("marketplaceId") == market]
+ if market and summaries and not matching:
+ continue
+ asin = row.get("asin") or next((item.get("asin") for item in matching if item.get("asin")), None)
+ for issue in issues:
+ if not isinstance(issue, dict) or not (issue.get("message") or issue.get("code")):
+ incomplete = True
+ continue
+ related.append({"asin": asin, "sku": row.get("sku"),
+ "issue": issue.get("message") or issue["code"],
+ "severity": issue.get("severity")})
+ after = page["nextCursor"]
+ if after is None:
+ break
+ if cursor is not None and int(after) <= int(cursor):
+ raise RuntimeError("The listing source cursor did not advance.")
+ cursor = after
+ except perm_scope.Denied:
+ return [], "Related listings are unavailable with your current source access."
+ except (perm_scope.UnknownTable, perm_scope.Unresolvable):
+ return [], "Related listing data is unavailable."
+ return related, "Some listing issues are missing or unreadable." if incomplete else None
+
+
+def category_detail(tenant_key, key, *, user=None, allowed=None, st=None):
+ """Serve one permitted category and separate listing evidence without claiming violation records."""
+ st = _rt(tenant_key, st)
+ category, _sep, _market = str(key).partition("@")
+ labels = dict(COUNT_METRICS)
+ if category not in labels:
+ raise KeyError(key)
+ kept, offset, total = [], 0, 0
+ while True:
+ page = summary_page(tenant_key, category=category, metric=key if _sep else None,
+ offset=offset, limit=PAGE_SIZE, st=st)
+ total = page["metricTotal"]
+ kept.extend(row for row in page["metrics"] if allowed is None or allowed(row, page["snapshotDate"]))
+ if len(kept) > 1:
+ raise ValueError("Choose a marketplace with category@marketplace.")
+ if page["cursor"] is None:
+ break
+ offset = int(page["cursor"])
+ if not kept and (total or allowed is not None or _sep):
+ raise KeyError(key)
+ row = kept[0] if kept else {}
+ note = CATEGORY_SOURCE_NOTE
+ if not row:
+ note += " This category is missing from the saved report."
+ if row and row.get("value") is None:
+ note += " The category count is missing."
+ if row and row.get("target") is None:
+ note += " Amazon did not provide a target."
+ rule = str(row.get("targetCondition") or "").upper()
+ known_rule = "LESS" in rule or "GREATER" in rule
+ if row.get("target") is not None and not known_rule:
+ note += " The target rule is missing or unsupported."
+ if row and (row.get("windowFrom") is None or row.get("windowTo") is None):
+ note += " The reporting window is missing or incomplete."
+ related, listing_note = _related_listings(tenant_key, st, user, row.get("marketplaceId")) if row else ([], None)
+ if listing_note:
+ note += " " + listing_note
+ query = urlencode({"marketplaceID": row["marketplaceId"]}) if row.get("marketplaceId") else ""
+ status = row.get("status", "No data")
+ if row.get("value") is not None and row.get("target") is not None and not known_rule:
+ status = "No target"
+ return {"key": category, "label": labels[category], "count": row.get("value"),
+ "status": status, "target": row.get("condition") if known_rule else None,
+ "window": {"from": row.get("windowFrom"), "to": row.get("windowTo")},
+ "recordList": False, "sourceNote": note,
+ "sellerCentralUrl": "https://sellercentral.amazon.com/performance/dashboard" + ("?" + query if query else ""),
+ "related": related}
+
+
# --------------------------------------------------------------------------- settings and owner
#: Accounts that sign in for a process, never a person who can own an alert (same list as amazon_profit).
@@ -919,7 +1031,7 @@ def _title(row):
return f"{row['label']} is {'at risk' if row['status'] == 'At risk' else 'near target'}"
-def _alert_source(tenant_key, st, row, snapshot_date):
+def _alert_source(tenant_key, st, row, snapshot_date, *, extra_fields=()):
"""Bind an alert to the actual mirror row supplying its metric value."""
from harness.amazon_schema import IDENTITY_PID
column = row["key"] + ("_defects_count" if row["key"] in dict(COUNT_METRICS) else "_rate")
@@ -930,6 +1042,7 @@ def _alert_source(tenant_key, st, row, snapshot_date):
return {"source": None, "fields": fields}
fields += [name for name in (row["key"] + "_target_value", row["key"] + "_target_condition")
if name in cols]
+ fields += [name for name in extra_fields if name in cols and name not in fields]
order = ' ORDER BY "_report_row_number"' if "_report_row_number" in cols else ""
row_ids = []
for field in (name for name in fields if name not in {"_snapshot_date", "marketplace_id"}):
@@ -945,6 +1058,14 @@ def _alert_source(tenant_key, st, row, snapshot_date):
"fields": fields}
+def category_source(tenant_key, st, row, snapshot_date):
+ """Bind every category detail field to the report rows that supplied it."""
+ key = row["key"]
+ return _alert_source(tenant_key, st, row, snapshot_date, extra_fields=(
+ key + "_status", key + "_reporting_date_range_reporting_date_from",
+ key + "_reporting_date_range_reporting_date_to"))
+
+
def raise_alerts(tenant_key, view=None, *, st=None, now=None):
"""Raise one owned alert per metric that is near or past its target, never a second while one is open."""
st = _rt(tenant_key, st)
@@ -1731,7 +1852,7 @@ def validate_dashboard(tenant_key, figures=None, *, start=None, end=None, st=Non
return {"ok": bool(detail) and all(check["ok"] for check in detail), "detail": detail}
def validate(tenant_key, *, st=None, **period):
- """Tie the rating and each rate in `summary` to the raw `ut_amzn_account_health` snapshot."""
+ """Tie the rating, rates and category detail counts to the raw account health snapshot."""
st = _rt(tenant_key, st)
view = summary(tenant_key, st=st)
with _reader(tenant_key, st) as cur:
@@ -1742,17 +1863,33 @@ def validate(tenant_key, *, st=None, **period):
date = cur.execute(f'SELECT MAX(CAST("_snapshot_date" AS VARCHAR)) FROM {AH_TABLE}').fetchone()[0] \
if "_snapshot_date" in cols else None
wanted = [("rating", "account_health_rating_ahr_score", 1.0)] + \
- [(p, f"{p}_rate", None) for p, _l, _d in RATE_METRICS]
+ [(p, f"{p}_rate", None) for p, _l, _d in RATE_METRICS] + \
+ [(p, f"{p}_defects_count", 1.0) for p, _l in COUNT_METRICS] + [(WARNINGS[0], WARNINGS[2], 1.0)]
for key, col, _s in wanted:
if col not in cols:
continue
where = ' WHERE CAST("_snapshot_date" AS VARCHAR) = ?' if date is not None else ""
order = ' ORDER BY "_report_row_number"' if "_report_row_number" in cols else ""
- q = (f'SELECT CAST("marketplace_id" AS VARCHAR), FIRST(TRY_CAST("{col}" AS DOUBLE){order}) '
+ q = (f'SELECT CAST("marketplace_id" AS VARCHAR), '
+ f'FIRST(TRY_CAST(REPLACE(TRIM(CAST("{col}" AS VARCHAR)), \',\', \'\') AS DOUBLE){order}) '
f'FILTER (WHERE "{col}" IS NOT NULL) '
f'FROM {AH_TABLE}{where} GROUP BY 1')
for market, value in cur.execute(q, [date] if date is not None else []).fetchall():
independent[(key, str(market or ""))] = value
+ inbound_cols = _columns(cur, INBOUND_TABLE)
+ if inbound_cols:
+ column = next((name for name in ("issue_reported_date", "_period_start") if name in inbound_cols), None)
+ where, args = "", []
+ if column:
+ try:
+ end = dt.date.fromisoformat(str(date)[:10])
+ except (TypeError, ValueError):
+ end = _now().date()
+ where = (f' WHERE TRY_CAST(SUBSTR(CAST("{column}" AS VARCHAR), 1, 10) AS DATE) '
+ f'> ? AND TRY_CAST(SUBSTR(CAST("{column}" AS VARCHAR), 1, 10) AS DATE) <= ?')
+ args = [end - dt.timedelta(days=INBOUND_DAYS), end]
+ independent[(INBOUND[0], "")] = cur.execute(
+ f"SELECT COUNT(*) FROM {INBOUND_TABLE}{where}", args).fetchone()[0]
app = {}
for m in view["marketplaces"]:
app[("rating", m["marketplaceId"] or "")] = m["rating"]
@@ -1761,6 +1898,8 @@ def validate(tenant_key, *, st=None, **period):
# The shown percent ties back through its scale; a no orders row ties its raw rate (Amazon's 0).
raw = row["value"] / row.get("scale", 100.0) if row["value"] is not None else row.get("raw")
app[(row["key"], row["marketplaceId"] or "")] = raw
+ else:
+ app[(row["key"], row["marketplaceId"] or "")] = row["value"]
detail, app_total, ind_total = [], 0.0, 0.0
for key in sorted(set(app) | set(independent)):
a, b = app.get(key), independent.get(key)
@@ -1770,9 +1909,24 @@ def validate(tenant_key, *, st=None, **period):
detail.append({"metric": key[0], "marketplaceId": key[1] or None, "app": a, "independent": b, "ok": tied})
ok = bool(detail) and all(d["ok"] for d in detail) and str(view.get("snapshotDate")) == str(date)
groups = _validate_groups(tenant_key, st, view)
- return {"ok": ok and reserves["ok"] and groups["ok"], "app_total": round(app_total, 9),
+ categories = []
+ for key, market in sorted(set(app) | set(independent)):
+ if key not in dict(COUNT_METRICS):
+ continue
+ try:
+ shown = category_detail(tenant_key, key + ("@" + market if market else ""), st=st)["count"]
+ source = independent.get((key, market))
+ tied = (shown is None and source is None) or (shown is not None and source is not None
+ and abs(shown - source) <= 1e-7)
+ except KeyError:
+ shown, source = None, independent.get((key, market))
+ tied = (key, market) not in app and source is None
+ categories.append({"metric": key, "marketplaceId": market or None,
+ "app": shown, "independent": source, "ok": tied})
+ category_check = {"ok": all(row["ok"] for row in categories), "detail": categories}
+ return {"ok": ok and reserves["ok"] and groups["ok"] and category_check["ok"], "app_total": round(app_total, 9),
"independent_total": round(ind_total, 9), "detail": detail, "snapshotDate": date,
- "message": None if detail else EMPTY, "reserves": reserves, "groups": groups}
+ "message": None if detail else EMPTY, "reserves": reserves, "groups": groups, "categories": category_check}
def _validate_groups(tenant_key, st, view):
diff --git a/platform/modules/amazon_profit.py b/platform/modules/amazon_profit.py
index d733d83f6e86d60b38cc00f3e379b89ba8a41aa9..a4fb9a4e7f8436ae9ae75cf53316310af55a46d6 100644
--- a/platform/modules/amazon_profit.py
+++ b/platform/modules/amazon_profit.py
@@ -10,9 +10,9 @@ survive every period switch.
`ut_amazon_asin_profit` and `ut_amazon_parent_profit` roll the SKU rows up on every refresh.
Projects retain their own verified period totals and Impact in the tenant projects store.
-`validate()` ties a month's net proceeds to Amazon settlements within 2% over the days its issued
-settlements cover, excluding loan repayments and withheld tax, and proves a negative control that
-drops fees goes red. A refresh runs it for the last complete month and writes no figures when it
+`validate()` ties each net proceeds component to independent Amazon amounts within 2% over settled
+days, with order charges on purchase dates and monthly storage in its service month.
+A refresh runs it for the last complete month and writes no figures when it
fails. A passing refresh stores the verified range and the day the rest is due to settle.
Owned items (R10, R31): a refresh opens one item on a Losing or Thin SKU with no status, for the
@@ -615,11 +615,16 @@ def reported_net_proceeds(con, start, end):
def economics_currency(con):
if "currency_code" not in _columns(con, ECONOMICS):
return None
- return con.execute(f"SELECT any_value(currency_code) FROM {ECONOMICS}").fetchone()[0]
+ currency, count, missing = con.execute(
+ f"SELECT max(currency_code), count(DISTINCT currency_code), "
+ f"count(*) FILTER (WHERE NULLIF(trim(currency_code), '') IS NULL) FROM {ECONOMICS}").fetchone()
+ if count > 1 or missing:
+ raise SourceUnavailable("Amazon economics needs one known currency.")
+ return currency
def settlements_total(con, start, end, currency=None):
- """Settlements posted in the window, excluding loan repayment and withheld tax.
+ """Sum posted settlements with both collected and withheld sales tax included.
`currency` keeps settlements whose header row names it: Amazon.ca and Amazon.com.mx settle
in their own currency, outside the economics marketplace. Their reports write DD.MM.YYYY.
@@ -629,18 +634,188 @@ def settlements_total(con, start, end, currency=None):
stream = ("JOIN (SELECT settlement_id FROM {t} WHERE currency IS NOT NULL GROUP BY settlement_id "
"HAVING any_value(currency) = ?) h USING (settlement_id)").format(t=SETTLEMENTS) if scoped else ""
total, rows, undated = con.execute(
- "SELECT COALESCE(SUM(amount) FILTER (WHERE d BETWEEN CAST(? AS DATE) AND CAST(? AS DATE)), 0), "
+ "SELECT COALESCE(SUM(TRY_CAST(amount AS DECIMAL(38, 2))) "
+ "FILTER (WHERE d BETWEEN CAST(? AS DATE) AND CAST(? AS DATE)), 0), "
"count(*) FILTER (WHERE d BETWEEN CAST(? AS DATE) AND CAST(? AS DATE)), "
"count(*) FILTER (WHERE d IS NULL AND posted_date IS NOT NULL) "
"FROM (SELECT amount, posted_date, COALESCE(TRY_CAST(left(posted_date, 10) AS DATE), "
"CAST(try_strptime(left(posted_date, 10), '%d.%m.%Y') AS DATE)) AS d "
- f"FROM {SETTLEMENTS} s {stream} WHERE COALESCE(transaction_type, '') <> ? "
- "AND COALESCE(amount_type, '') <> ?)",
- [start, end, start, end] + ([currency] if scoped else []) + [LOAN_REPAYMENT, WITHHELD_TAX]).fetchone()
+ f"FROM {SETTLEMENTS} s {stream} WHERE COALESCE(transaction_type, '') <> ?)",
+ [start, end, start, end] + ([currency] if scoped else []) + [LOAN_REPAYMENT]).fetchone()
return {"total": float(total), "rows": int(rows), "undated": int(undated),
"currency": currency if scoped else None}
+def settlement_components(con, start, end, currency=None):
+ """Tie purchase-date order charges, service-month storage and daily ads to independent sources."""
+ month = dt.date.fromisoformat(start).strftime("%B %Y")
+ parts, problems = [], []
+
+ def problem(component, reason):
+ problems.append(f"Not verified: {component} in {month}: {reason}.")
+
+ def tie(component, ours, source, *, reason=None):
+ ours = float(ours)
+ source = None if source is None else float(source)
+ ok, gap = _tie(ours, source) if source is not None else (False, math.inf)
+ basis = {"product sales": "order purchase date", "order fees": "order purchase date",
+ "monthly storage": "storage service month", "ads": "daily ad report date"}
+ parts.append({"component": component, "month": start[:7], "ok": ok and not reason,
+ "a": ours, "b": source, "gap": gap, "tolerance": SETTLEMENT_TOLERANCE,
+ "dateBasis": basis.get(component, "order purchase date when present, otherwise posted date"),
+ "sourceTables": ([ECONOMICS] + [f"amzn_ads_{kind}_advertised_products" for kind in ("sp", "sb", "sd")]
+ if component == "ads" else [ECONOMICS, SETTLEMENTS, "amzn_orders"])})
+ if reason:
+ problem(component, reason)
+ elif not ok:
+ problem(component, f"independent Amazon amounts differ by {gap:.1%}")
+
+ sc = _columns(con, SETTLEMENTS)
+ ec = _columns(con, ECONOMICS)
+ required = set(_SETTLEMENT_REQUIRED) | {"settlement_id", "currency", "order_id", "amount_description"}
+ if not required <= sc:
+ problem("settlement components", "missing source columns " + ", ".join(sorted(required - sc)))
+ return {"total": None, "parts": parts, "reason": problems[0], "rows": 0}
+ if currency is None:
+ problem("settlement components", "economics currency unavailable")
+ return {"total": None, "parts": parts, "reason": problems[0], "rows": 0}
+ if not {"amazon_order_id", "purchase_date"} <= _columns(con, "amzn_orders"):
+ problem("order charges", "missing amzn_orders purchase dates")
+ return {"total": None, "parts": parts, "reason": problems[0], "rows": 0}
+ unscoped = con.execute(f"""SELECT count(*) FROM {SETTLEMENTS} s
+ LEFT JOIN (SELECT settlement_id FROM {SETTLEMENTS} GROUP BY settlement_id
+ HAVING count(DISTINCT currency) = 1) h USING (settlement_id)
+ WHERE h.settlement_id IS NULL AND (s.amount IS NOT NULL OR NULLIF(s.amount_type, '') IS NOT NULL)
+ AND (CAST({_moment_sql('posted_date')} AS DATE) BETWEEN CAST(? AS DATE) AND CAST(? AS DATE)
+ OR {_moment_sql('posted_date')} IS NULL)""", [start, end]).fetchone()[0]
+ if unscoped:
+ problem("settlement currency", "missing or conflicting settlement header currency")
+ # Aggregate in SQL before reading: duplicate order lines must not multiply settlement amounts.
+ cte = f"""WITH orders AS (
+ SELECT amazon_order_id, min(CAST({_moment_sql('purchase_date')} AS DATE)) AS bought,
+ count(DISTINCT CAST({_moment_sql('purchase_date')} AS DATE)) AS dates,
+ count(*) FILTER (WHERE {_moment_sql('purchase_date')} IS NULL) AS bad_dates
+ FROM amzn_orders GROUP BY amazon_order_id),
+ streams AS (SELECT settlement_id FROM {SETTLEMENTS} GROUP BY settlement_id
+ HAVING count(DISTINCT currency) = 1 AND max(currency) = ?),
+ lines AS (SELECT s.*, TRY_CAST(amount AS DECIMAL(38, 2)) AS currency_amount,
+ CAST({_moment_sql('s.posted_date')} AS DATE) AS posted,
+ o.bought, o.dates, o.bad_dates,
+ lower(regexp_replace(COALESCE(amount_description, ''), '[^a-zA-Z0-9]', '', 'g')) AS description,
+ lower(regexp_replace(COALESCE(amount_type, ''), '[^a-zA-Z0-9]', '', 'g')) AS kind,
+ lower(regexp_replace(COALESCE(transaction_type, ''), '[^a-zA-Z0-9]', '', 'g')) AS transaction
+ FROM {SETTLEMENTS} s JOIN streams USING (settlement_id)
+ LEFT JOIN orders o ON trim(s.order_id) = o.amazon_order_id
+ WHERE (amount IS NOT NULL OR NULLIF(amount_type, '') IS NOT NULL)
+ AND COALESCE(transaction_type, '') <> ?),
+ classified AS (SELECT *, CASE
+ WHEN concat(kind, description, transaction) LIKE '%inbounddefect%' THEN 'Inbound Defect Fee'
+ WHEN concat(kind, description, transaction) LIKE '%mcf%' THEN 'MCF credits'
+ WHEN kind = 'itemwithheldtax' OR description IN ('tax', 'shippingtax', 'giftwraptax')
+ OR description LIKE 'marketplacefacilitatortax%' THEN 'tax'
+ WHEN (kind = 'itemprice' AND (description IN ('shipping', 'giftwrap')
+ OR description LIKE '%promotion%')) OR kind IN ('promotion', 'shipping', 'shippingfees')
+ THEN 'shipping and promotions'
+ WHEN 'costofadvertising' IN (kind, description, transaction)
+ OR description IN ('advertising', 'advertisingfee') THEN 'ads billing'
+ WHEN description IN ('storagefee', 'fbamonthlystoragefee', 'monthlystoragefee')
+ OR 'fbainventorystoragefee' IN (kind, description, transaction) THEN 'monthly storage'
+ WHEN concat(kind, description, transaction) LIKE '%reimbursement%' THEN 'reimbursements'
+ WHEN description IN ('subscription', 'subscriptionfee', 'fbaremovalfee', 'fbadisposalfee',
+ 'fbainboundtransportationfee', 'couponredemptionfee', 'dealfee')
+ OR kind IN ('dealfee', 'lightningdealfee', 'dealparticipationfee', 'dealperformancefee',
+ 'fbacustomerreturnfee', 'fbacustomerreturnsfee', 'fbacustomerreturnperunitfee',
+ 'fbacustomerreturnprocessingfee', 'fbainboundplacementservicefee')
+ OR description IN ('lightningdealfee', 'dealparticipationfee', 'dealperformancefee',
+ 'fbacustomerreturnfee', 'fbacustomerreturnsfee', 'fbacustomerreturnperunitfee',
+ 'fbacustomerreturnprocessingfee', 'fbainboundplacementservicefee')
+ OR transaction IN ('dealfee', 'lightningdealfee', 'dealparticipationfee', 'dealperformancefee',
+ 'fbacustomerreturnfee', 'fbacustomerreturnsfee', 'fbacustomerreturnperunitfee',
+ 'fbacustomerreturnprocessingfee', 'fbainboundplacementservicefee') THEN 'account fees'
+ WHEN NULLIF(trim(order_id), '') IS NOT NULL AND kind = 'itemprice'
+ AND description = 'principal' THEN 'product sales'
+ WHEN NULLIF(trim(order_id), '') IS NOT NULL AND kind IN ('itemfee', 'itemfees') THEN 'order fees'
+ ELSE 'unmatched component' END AS component FROM lines),
+ dated AS (SELECT *, CASE
+ WHEN component = 'monthly storage' THEN CAST(date_trunc('month', posted) - INTERVAL '1 month' AS DATE)
+ WHEN NULLIF(trim(order_id), '') IS NOT NULL THEN bought
+ ELSE posted END AS basis FROM classified)
+ """
+ groups = con.execute(cte + """SELECT component, COALESCE(SUM(currency_amount), 0), count(*),
+ count(*) FILTER (WHERE currency_amount IS NULL OR posted IS NULL),
+ count(*) FILTER (WHERE NULLIF(trim(order_id), '') IS NOT NULL
+ AND (bought IS NULL OR dates <> 1 OR bad_dates <> 0))
+ FROM dated WHERE basis BETWEEN CAST(? AS DATE) AND CAST(? AS DATE)
+ OR ((basis IS NULL OR dates <> 1 OR bad_dates <> 0)
+ AND (posted BETWEEN CAST(? AS DATE) AND CAST(? AS DATE) OR posted IS NULL))
+ GROUP BY component ORDER BY component""",
+ [currency, LOAN_REPAYMENT, start, end, start, end]).fetchall()
+ amounts = {name: total for name, total, *_ in groups}
+ if not groups:
+ problem("settlement components", "no settlement source rows on the purchase or service dates")
+ for name, _total, _n, bad, unmatched in groups:
+ if bad or unmatched:
+ problem(name, "unmatched purchase date, unreadable posting date or unavailable amount")
+ if name in ("unmatched component", "Inbound Defect Fee", "MCF credits"):
+ problem(name, "settlement category has no economics mapping")
+ fee_columns = sorted(c for c in ec if c.startswith("fee_") and c.endswith("_amount") and "_charge_" not in c)
+ storage_columns = [c for c in fee_columns if "storage" in c]
+ reimbursement_columns = [c for c in fee_columns if "reimbursement" in c]
+ account_columns = [c for c in fee_columns if any(name in c for name in
+ ("subscription", "removal", "disposal", "inbound_transportation", "inbound_placement",
+ "coupon", "deal_", "customer_return", "return_processing", "returns_processing"))]
+ # Storage fee columns are accruals in economics; only monthly settlement charges use the prior month.
+ sums = ["sales_net_product_sales_amount", "fees_total_amount", *ads_columns(ec),
+ *storage_columns, *reimbursement_columns, *account_columns]
+ observed = con.execute("SELECT " + ", ".join(f'COALESCE(SUM(TRY_CAST("{c}" AS DECIMAL(38, 2))), 0)' for c in sums)
+ + ", count(*) FILTER (WHERE "
+ + " OR ".join(f'("{c}" IS NOT NULL AND TRY_CAST("{c}" AS DECIMAL(38, 2)) IS NULL)'
+ for c in sums) + ")"
+ + f" FROM {ECONOMICS} WHERE left(start_date, 10) BETWEEN ? AND ?",
+ [start, end]).fetchone()
+ if observed[-1]:
+ problem("settlement components", "unavailable economics amount")
+ values = dict(zip(sums, observed[:-1]))
+ sales, fees = values[sums[0]], values[sums[1]]
+ storage = sum(values[c] for c in storage_columns)
+ reimbursements = sum(values[c] for c in reimbursement_columns)
+ account_fees = sum(values[c] for c in account_columns)
+ ads = sum(values[c] for c in ads_columns(ec))
+ tie("product sales", sales, amounts.get("product sales", 0.0))
+ tie("order fees", fees - storage - reimbursements - account_fees, -amounts.get("order fees", 0.0))
+ tie("monthly storage", storage, -amounts.get("monthly storage", 0.0))
+ tie("reimbursements", reimbursements, -amounts.get("reimbursements", 0.0))
+ tie("account fees", account_fees, -amounts.get("account fees", 0.0))
+ tie("sales tax", 0.0, amounts.get("tax", 0.0))
+ # Shipping and promotions lie outside net product sales; ad invoices use a separate billing period.
+ if ads or "ads billing" in amounts:
+ ad_total = 0
+ for kind in ("sp", "sb", "sd"):
+ table = f"amzn_ads_{kind}_advertised_products"
+ if not {"date", "cost"} <= _columns(con, table):
+ problem("ads", f"missing source {table}.cost")
+ ad_total = None
+ break
+ total, count, present = con.execute(
+ f"SELECT SUM(TRY_CAST(cost AS DECIMAL(38, 2))), count(*), "
+ f"count(TRY_CAST(cost AS DECIMAL(38, 2))) FROM {table} "
+ "WHERE left(CAST(date AS VARCHAR), 10) BETWEEN ? AND ?", [start, end]).fetchone()
+ if not count or count != present or total is None or not math.isfinite(float(total)):
+ problem("ads", f"unavailable daily amounts in {table}")
+ ad_total = None
+ break
+ ad_total += total
+ tie("ads", ads, ad_total)
+ else:
+ ad_total = 0
+ target = (float(sum(amounts.get(name, 0) for name in
+ ("product sales", "order fees", "monthly storage", "reimbursements", "account fees")))
+ - float(ad_total)) if ad_total is not None else None
+ return {"total": target, "parts": parts, "reason": problems[0] if problems else None,
+ "rows": sum(row[2] for row in groups),
+ "excluded": {k: float(amounts[k]) for k in ("shipping and promotions", "ads billing") if k in amounts}}
+
+
_COVERAGE_REQUIRED = ("settlement_id", "total_amount", "settlement_start_date", "settlement_end_date")
@@ -1988,6 +2163,8 @@ def refusal(checks, month):
)
for index, check in enumerate(checks):
if not check.get("ok"):
+ if index == 0 and check.get("reason"):
+ return check["reason"]
if index < len(sentences):
return sentences[index](check)
return f"Not verified: the Amazon figures do not tie in {label}."
@@ -2685,7 +2862,9 @@ def _mirror_has_economics(con):
return False
-def snapshot_cursor(tenant=TENANT_ZERO, tables=(ECONOMICS, SETTLEMENTS, LISTINGS)):
+def snapshot_cursor(tenant=TENANT_ZERO, tables=(ECONOMICS, SETTLEMENTS, LISTINGS, "amzn_orders",
+ "amzn_ads_sp_advertised_products", "amzn_ads_sb_advertised_products",
+ "amzn_ads_sd_advertised_products")):
"""An in-memory DuckDB over the stored mirror snapshot. Downloads only; writes nothing back."""
key = (tenant, tuple(tables))
if key in _SNAPSHOT:
@@ -2820,7 +2999,7 @@ def typed_cogs_checks(con, computed, typed, usage=None, bounds=None):
def validate(pre=None, month=None, con=None, *, cost_usage=None, cost_history=None, typed_rows=None,
typed_values=None, typed_period=None):
- """validate.py convention: net proceeds tie to settlements within 2% over the month's settled days.
+ """Tie net proceeds components to independent Amazon sources within 2% over settled days.
Receipt COGS ties to Odoo; typed COGS (`typed_rows`) is listed apart and ties to its stored values."""
month = month or last_complete_month()
bounds = period_bounds(month)
@@ -2829,25 +3008,35 @@ def validate(pre=None, month=None, con=None, *, cost_usage=None, cost_history=No
if con is None:
con, source = validation_cursor()
checks = []
- currency = economics_currency(con)
- span = settlement_coverage(con, start, end, currency)
- # Both sides tie the same covered days: economics by start_date, settlement lines by posted_date.
+ try:
+ _require(con, ECONOMICS, _ECONOMICS_REQUIRED)
+ currency = economics_currency(con)
+ span = settlement_coverage(con, start, end, currency)
+ settled = settlements_total(con, start, end, currency)
+ except SourceUnavailable as exc:
+ label = dt.date.fromisoformat(start).strftime("%B %Y")
+ raise SourceUnavailable(f"Not verified: settlement components in {label}: {exc}") from None
lo, hi = span["start"] or start, span["end"] or end
proceeds = net_proceeds_total(con, lo, hi)
- settled = settlements_total(con, lo, hi, currency)
+ aligned = settlement_components(con, lo, hi, currency)
if span["start"] is None:
- settled = dict(settled, total=0.0, rows=0)
- ok, gap = _tie(proceeds, settled["total"])
+ aligned = dict(aligned, total=None, rows=0,
+ reason=f"Not verified: settlement components in {bounds['label']}: no settled days.")
+ target = aligned["total"]
+ ok, gap = _tie(proceeds, target) if target is not None else (False, math.inf)
covered = {k: span[k] for k in ("start", "end", "label", "partial", "clear", "sentence")}
checks.append({"check": f"amazon_profit {month}: net proceeds tie to settlements within 2% over "
f"{span['label'] or 'no settled days'} ({gap:.2%} apart, {source})",
- "ok": ok, "a": round(proceeds, 2), "b": round(settled["total"], 2),
- "gap": round(gap, 4), "covered": covered})
+ "ok": ok and not aligned["reason"], "a": round(proceeds, 2),
+ "b": None if target is None else round(target, 2), "reason": aligned["reason"],
+ "components": aligned["parts"], "excluded": aligned.get("excluded", {}),
+ "gap": round(gap, 4), "covered": covered, "tolerance": SETTLEMENT_TOLERANCE})
dropped = net_proceeds_total(con, lo, hi, drop_fees=True)
- control_ok, control_gap = _tie(dropped, settled["total"])
+ control_ok, control_gap = _tie(dropped, target) if target is not None else (False, math.inf)
checks.append({"check": f"amazon_profit {month}: NC net proceeds without fees miss settlements "
f"({control_gap:.2%} apart)",
- "ok": not control_ok, "a": round(dropped, 2), "b": round(settled["total"], 2),
+ "ok": not control_ok, "a": round(dropped, 2),
+ "b": None if target is None else round(target, 2),
"gap": round(control_gap, 4)})
checks.append({"check": f"amazon_profit {month}: every {settled['currency'] or 'settlement'} "
"settlement row carries a readable date",
diff --git a/platform/modules/procurement.py b/platform/modules/procurement.py
index 46b1464b2ed257646789d4a5f9ebccf70e4564be..18269c67201e1ae5ca0c02a51daf031d6b9e98fc 100644
--- a/platform/modules/procurement.py
+++ b/platform/modules/procurement.py
@@ -20,6 +20,7 @@ READ-ONLY on Odoo; the team's Order Qty + master-data edits persist in the platf
"""
import sys
import json
+import hashlib
import math
import calendar
import datetime as dt
@@ -153,6 +154,22 @@ def _preorders(cutoff_iso):
return qty, need
+def _fba_refusal(product_id, quantity, report):
+ """Classify unknown FBA stock without treating missing or invalid quantities as zero."""
+ if not product_id:
+ return {'cause_code': 'no_product_id', 'cause': 'no product id for this SKU'}
+ if report.get('available') is False:
+ cause_code = ('non_numeric_fba' if report.get('cause_code') == 'non_numeric_fba'
+ else 'report_unavailable')
+ return {'cause_code': cause_code,
+ 'cause': report.get('cause') or 'FBA stock report unavailable'}
+ if (not isinstance(quantity, (int, float)) or isinstance(quantity, bool)
+ or not math.isfinite(quantity)):
+ return {'cause_code': 'non_numeric_fba',
+ 'cause': 'FBA quantity is missing or is not a finite number for this product'}
+ return None
+
+
def recommendations(t=None):
"""Base per-SKU table (curated master data + live Odoo demand/stock/PO + alt-vendors). The UI
overlays the team's vendor/country/cost edits on top; reorder methods + CBM/container are here."""
@@ -191,13 +208,13 @@ def recommendations(t=None):
try:
import modules.warehouse as warehouse
fba_by_pid, fba_report = warehouse.fba_qty_by_product_id(
- [p['id'] for p in by_code.values()])
+ [p.get('id') for p in by_code.values()])
except Exception as exc:
fba_by_pid, fba_report = None, {
'available': False, 'cause': 'the FBA warehouse read failed: ' + type(exc).__name__}
rows = []
for code, p in by_code.items():
- pid = p['id']
+ pid = p.get('id')
meta = sup.get(code, {})
lead = meta.get('lead') or 0
tmpl = O.m2o_id(p.get('product_tmpl_id'))
@@ -211,7 +228,8 @@ def recommendations(t=None):
on_hand = p.get('qty_available') or 0.0
po = inpo.get(pid, 0.0)
fba = (fba_by_pid or {}).get(pid)
- fba_known = isinstance(fba, (int, float)) and fba_report.get('available') is not False
+ fba_refusal = _fba_refusal(pid, fba, fba_report)
+ fba_known = fba_refusal is None
avail = on_hand - fba + po if fba_known else None
days_cover = ((avail / avg_day) if avg_day > 0 else
(None if avail <= 0 else 9999.0)) if avail is not None else None
@@ -245,8 +263,7 @@ def recommendations(t=None):
})
if not fba_known:
rows[-1]['_coverage_report'] = {**fba_report, 'available': False,
- 'cause': fba_report.get('cause')
- or 'FBA stock unavailable for this product'}
+ **fba_refusal}
return rows
@@ -649,13 +666,13 @@ def validate_coverage(rows, t=None, oracle=None, demand=None):
unavailable, wrong_days, wrong_buy = [], [], []
for r in rows:
pid = r.get('_product_id')
- known = (pid is not None and isinstance(r.get('_fba'), (int, float))
- and r.get('_coverage_report', {}).get('available') is not False)
+ report = r.get('_coverage_report', {})
+ refusal = _fba_refusal(pid, r.get('_fba'), report)
expected_days, expected_buy = None, None
- if not known:
+ if refusal is not None:
unavailable.append({'code': r['code'],
- 'cause': r.get('_coverage_report', {}).get('cause')
- or 'FBA stock unavailable for this product'})
+ 'sku_ref': hashlib.sha256(r['code'].encode('utf-8')).hexdigest(),
+ **refusal})
else:
effective = stock['non_fba_by_pid'].get(pid, 0.0) + float(r.get('incoming_po') or 0.0)
daily = demand.get(pid, 0.0) / window_days
diff --git a/platform/modules/returns_refunds.py b/platform/modules/returns_refunds.py
index d790012385aef56cf61d299b7274a953ecce7ac1..d22de26b16bd7ef980a7d1a7370821e29e188be7 100644
--- a/platform/modules/returns_refunds.py
+++ b/platform/modules/returns_refunds.py
@@ -1987,6 +1987,33 @@ def _return_source(cur, table, date, quantity, sku="sku", reason="reason"):
+ (f" FROM {table}" if cols else " WHERE false") + ") captured")
+def _refund_overlap_relations():
+ """Match wildcard identities through distinct projections with at most eight probes per identity."""
+ fields = ('order_id', 'sku', 'settlement_id')
+ mask = '+'.join(f'CASE WHEN {field} IS NULL THEN 0 ELSE {1 << i} END'
+ for i, field in enumerate(fields))
+ projection = ','.join(f"CASE WHEN (p.mask & {1 << i})!=0 THEN {field} ELSE '' END AS {field}"
+ for i, field in enumerate(fields))
+ matches = ' AND '.join(
+ f"b.{field}=CASE WHEN (a.mask & m.mask & {1 << i})!=0 THEN a.{field} ELSE '' END"
+ for i, field in enumerate(fields))
+ identities = ' AND '.join(f'a.{field} IS NOT DISTINCT FROM o.{field}' for field in fields)
+ return f"""refund_identities AS (
+ SELECT DISTINCT source,account,order_id,sku,settlement_id,{mask} AS mask FROM refund_candidates),
+ refund_overlap_keys AS (
+ SELECT DISTINCT source,account,i.mask AS identity_mask,p.mask AS projection_mask,{projection}
+ FROM refund_identities i CROSS JOIN range(8) p(mask) WHERE (i.mask & p.mask)=p.mask),
+ refund_overlaps AS (
+ SELECT a.source,a.account,a.order_id,a.sku,a.settlement_id,bool_or(b.source IS NOT NULL) AS overlap
+ FROM refund_identities a CROSS JOIN range(8) m(mask)
+ LEFT JOIN refund_overlap_keys b ON a.source!=b.source AND a.account=b.account
+ AND b.identity_mask=m.mask AND b.projection_mask=(a.mask & m.mask) AND {matches}
+ GROUP BY a.source,a.account,a.order_id,a.sku,a.settlement_id),
+ refund_events AS (SELECT a.*,COALESCE(a.sku,a.asin,'{NO_SKU}') AS key,o.overlap
+ FROM refund_candidates a JOIN refund_overlaps o ON a.source=o.source AND a.account=o.account
+ AND {identities})"""
+
+
def _refund_relations(cur):
"""Keep source identities, capture deduplication and ambiguous overlap inside SQL."""
sc, tc = (_columns(cur, table) for table in FINANCE_TABLES)
@@ -2046,24 +2073,20 @@ def _refund_relations(cur):
SELECT 'finance',account,settlement_id,event_id,item_id,order_id,sku,asin,day,NULL,money,currency,
conflict,event_id IS NULL OR position IS NULL OR NOT isfinite(position) OR position<0
OR position!=trunc(position) OR sku IS NULL AND asin IS NULL, true FROM finance_events),
- refund_events AS (SELECT a.*,COALESCE(a.sku,a.asin,'{NO_SKU}') AS key,
- EXISTS(SELECT 1 FROM refund_candidates b WHERE a.source!=b.source AND a.account=b.account
- AND (a.order_id=b.order_id OR a.order_id IS NULL OR b.order_id IS NULL)
- AND (a.sku=b.sku OR a.sku IS NULL OR b.sku IS NULL)
- AND (a.settlement_id=b.settlement_id OR a.settlement_id IS NULL OR b.settlement_id IS NULL)) AS overlap
- FROM refund_candidates a)"""
+ {_refund_overlap_relations()}"""
-def _refund_totals_sql(first, last, group=False):
+def _refund_totals_sql(first, last, group=False, *, windows=False):
"""Aggregate only event-grain refunds after source capture deduplication."""
- return ("SELECT " + ("key," if group else "") + "COALESCE(sum(units),0) AS refunded,"
+ return ("SELECT " + ("window_id," if windows else "key," if group else "") + "COALESCE(sum(units),0) AS refunded,"
"sum(money) AS refund_cost,min(currency) AS refund_currency,count(DISTINCT currency) AS currencies,"
"count(*) FILTER (WHERE currency IS NULL OR NOT regexp_full_match(currency,'[A-Z]{3}') "
"OR money IS NULL OR NOT isfinite(money) OR money<0) AS money_bad,"
"count(*) FILTER (WHERE identity_bad OR conflict OR overlap OR day IS NULL) AS identity_bad,"
"count(*) FILTER (WHERE unit_bad) AS unit_bad,count(*) FILTER (WHERE source='finance') AS finance_units,"
"count(*) FILTER (WHERE overlap) AS overlaps FROM refund_events "
- f"WHERE day BETWEEN DATE '{first}' AND DATE '{last}' OR day IS NULL" + (" GROUP BY key" if group else ""))
+ + ("JOIN windows ON day BETWEEN first AND last OR day IS NULL GROUP BY window_id" if windows else
+ f"WHERE day BETWEEN DATE '{first}' AND DATE '{last}' OR day IS NULL" + (" GROUP BY key" if group else "")))
def _rate_values(row, ready):
@@ -3873,25 +3896,51 @@ def _dashboard_windows(first, last):
def _dashboard_rates(cur, first, last, independent):
- if not independent:
- return _aggregate(cur, _relations(cur, first, last, {'risingRatio': 1.5, 'risingMinPct': 5}))
- # Independent scalar folds avoid the serving membership and formatter projections.
+ return _dashboard_rate_windows(cur, [(first, last)], independent)[0]
+
+
+def _dashboard_rate_windows(cur, windows, independent=False, *, ready=None):
+ """Fold total, comparison and monthly windows in one query over complete source captures."""
+ bounds = ','.join(f"({i},DATE '{a.isoformat()}',DATE '{b.isoformat()}')" for i, (a, b) in enumerate(windows))
merchant_sku = _first(_columns(cur, MFN_TABLE), 'merchant_sku', 'sku') or 'merchant_sku'
physical = _return_source(cur, RETURNS_TABLE, 'return_date', 'quantity') + ' UNION ALL ' + _return_source(
cur, MFN_TABLE, 'return_request_date', 'return_quantity', merchant_sku, 'return_reason')
shipments = _return_source(cur, SHIPPED_TABLE, 'shipment_date', 'quantity')
- operands = _mirror_one(cur, f"""WITH returns AS ({physical}),shipments AS ({shipments}),
- {_refund_relations(cur)} SELECT * FROM
- (SELECT COALESCE(sum(units),0) AS returned FROM returns WHERE day BETWEEN ? AND ?) r CROSS JOIN
- (SELECT COALESCE(sum(units),0) AS shipped FROM shipments WHERE day BETWEEN ? AND ?) s CROSS JOIN
- (SELECT COALESCE(sum(units),0) AS refunded,sum(money) AS refund_cost,
+ prefix = f"""WITH windows(window_id,first,last) AS (VALUES {bounds}),
+ returns AS ({physical}),shipments AS ({shipments}),{_refund_relations(cur)},"""
+ if independent:
+ # Separate scalar source folds do not reuse the serving aggregate or formatter.
+ sql = prefix + """r AS (SELECT window_id,COALESCE(sum(units),0) AS returned
+ FROM windows LEFT JOIN returns ON day>=first AND day<=last GROUP BY window_id),
+ s AS (SELECT window_id,COALESCE(sum(units),0) AS shipped
+ FROM windows LEFT JOIN shipments ON day>=first AND day<=last GROUP BY window_id),
+ f AS (SELECT window_id,COALESCE(sum(units),0) AS refunded,sum(money) AS refund_cost,
min(currency) AS refund_currency,count(DISTINCT currency) AS currencies,
count(*) FILTER (WHERE money IS NULL OR money<0 OR NOT isfinite(money)
- OR currency IS NULL OR NOT regexp_full_match(currency,'[A-Z]{{3}}')) AS money_bad,
+ OR currency IS NULL OR NOT regexp_full_match(currency,'[A-Z]{3}')) AS money_bad,
bool_or(identity_bad OR conflict OR overlap OR day IS NULL) AS identity_bad,
- bool_or(unit_bad) AS unit_bad FROM refund_events
- WHERE day>=? AND day<=? OR day IS NULL) f""", [first, last, first, last, first, last])
- return _independent_rate_values(operands, _rate_availability(cur))
+ bool_or(unit_bad) AS unit_bad FROM refund_events JOIN windows
+ ON day>=first AND day<=last OR day IS NULL GROUP BY window_id)
+ SELECT r.*,s.shipped,f.* EXCLUDE(window_id) FROM r JOIN s USING(window_id)
+ LEFT JOIN f USING(window_id) ORDER BY window_id"""
+ else:
+ sql = prefix + """events AS (
+ SELECT day,units AS returned,0.0 AS shipped FROM returns
+ UNION ALL SELECT day,0.0,units FROM shipments),
+ quantities AS (SELECT window_id,COALESCE(sum(returned),0) AS returned,
+ COALESCE(sum(shipped),0) AS shipped FROM windows LEFT JOIN events
+ ON day BETWEEN first AND last GROUP BY window_id),
+ refunds AS (""" + _refund_totals_sql(None, None, windows=True) + """)
+ SELECT quantities.*,refunds.* EXCLUDE(window_id) FROM quantities
+ LEFT JOIN refunds USING(window_id) ORDER BY window_id"""
+ ready = _rate_availability(cur) if ready is None else ready
+ result = cur.execute(sql)
+ names = [field[0] for field in result.description]
+ rows = []
+ while batch := result.fetchmany(MAX_PAGE_SIZE):
+ rows.extend(dict(zip(names, row)) for row in batch)
+ formatter = _independent_rate_values if independent else _rate_values
+ return [formatter(row, ready) for row in rows]
def _dashboard_figures(tenant_key, *, start, end, st=None, user=None, independent=False):
@@ -3901,12 +3950,12 @@ def _dashboard_figures(tenant_key, *, start, end, st=None, user=None, independen
total, previous, points, reasons, reasons_value = {}, {}, [], [], None
if allowed:
with _screen_reader(tenant_key, st) as (cur, _present):
- total = _dashboard_rates(cur, first, last, independent)
- previous = _dashboard_rates(cur, first - (last - first) - dt.timedelta(days=1),
- first - dt.timedelta(days=1), independent)
- points = [(a.isoformat(), _dashboard_rates(cur, a, b, independent))
- for a, b in _dashboard_windows(first, last)]
ready = _rate_availability(cur)
+ months = list(_dashboard_windows(first, last))
+ windows = [(first, last), (first - (last - first) - dt.timedelta(days=1),
+ first - dt.timedelta(days=1)), *months]
+ total, previous, *monthly = _dashboard_rate_windows(cur, windows, independent, ready=ready)
+ points = [(a.isoformat(), values) for (a, _), values in zip(months, monthly)]
if ready['returns'] and ready['merchantReturns']:
merchant_sku = _first(_columns(cur, MFN_TABLE), 'merchant_sku', 'sku') or 'merchant_sku'
source = _return_source(cur, RETURNS_TABLE, 'return_date', 'quantity') + ' UNION ALL ' + _return_source(
@@ -3986,19 +4035,19 @@ def validate_figure(tenant_key, figure, *, start, end, st=None, user=None):
series += [dict(label=reason_label(reason), value=units) for reason, units in batch]
value, unit, prior = sum(row['value'] for row in series), 'returns', None
else:
- total = _dashboard_rates(cur, first, last, True)
+ months = list(_dashboard_windows(first, last))
+ windows = [(first, last), (first - (last - first) - dt.timedelta(days=1),
+ first - dt.timedelta(days=1)), *months]
+ total, previous, *monthly = _dashboard_rate_windows(cur, windows, True)
value = total[key]
unit = total['currency'] if key == 'refundCost' else '%'
if value is None or not unit:
return failed
- previous = _dashboard_rates(cur, first - (last - first) - dt.timedelta(days=1),
- first - dt.timedelta(days=1), True)
prior = previous[key]
if key == 'refundCost' and previous['currency'] != unit:
prior = None
series = []
- for begin, stop in _dashboard_windows(first, last):
- row = _dashboard_rates(cur, begin, stop, True)
+ for (begin, _), row in zip(months, monthly):
if key == 'refundCost' and row[key] is not None and row['currency'] != unit:
return failed
series.append(dict(x=begin.isoformat(), y=row[key]))
diff --git a/platform/modules/warehouse.py b/platform/modules/warehouse.py
index 9c41985b6005903b30d2516f572ad700ab288198..fbe064439f830746bd21c6c97dd6da3841b0d7c6 100644
--- a/platform/modules/warehouse.py
+++ b/platform/modules/warehouse.py
@@ -13,6 +13,7 @@ reconciles row-built aggregates to independent read_group/search_count paths.
"""
import sys
import datetime as dt
+import math
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
import core.odoo as O
@@ -792,9 +793,14 @@ def fba_qty_by_product_id(product_ids=None, wh=None):
for i in range(0, len(ids), _FBA_BATCH):
batches += 1
chunk = ids[i:i + _FBA_BATCH]
- for r in o.execute_kw('product.product', 'search_read', [[('id', 'in', chunk)]],
+ # Explicit selected ids include archived products; invalid quantities stay unknown.
+ for r in o.execute_kw('product.product', 'search_read',
+ [[('id', 'in', chunk), ('active', 'in', [True, False])]],
{'fields': ['id', 'qty_available'], 'context': ctx}):
- out[r['id']] = r.get('qty_available') or 0.0
+ quantity = r.get('qty_available')
+ if (isinstance(quantity, (int, float)) and not isinstance(quantity, bool)
+ and math.isfinite(quantity)):
+ out[r['id']] = quantity
return out, fba_report(wh, len(ids), len(out), batches,
'stock.warehouse.code at runtime, never a pinned id')
diff --git a/web/public/AIOS_DEPLOY_RECEIPT b/web/public/AIOS_DEPLOY_RECEIPT
index 2e92b77aa600edaf36bc50338c15e4a6fe5ab8bf..5ec71af105dc3e4d407a68d0a99d454ee8a25b1c 100644
--- a/web/public/AIOS_DEPLOY_RECEIPT
+++ b/web/public/AIOS_DEPLOY_RECEIPT
@@ -1 +1 @@
-10c06501f5337d94cc128c03b1fd10849aea6e293cf1290eb238d7c65538140d
+b50b579feefedb216021592e1ddd9160f4c6539ccea7f0bfbc906ab7944626c5
diff --git a/web/src/apps/AppRail.tsx b/web/src/apps/AppRail.tsx
index 5bd2c248bb47c766edbf2ff1944827219bac565d..52622d18674a4c32802b7cb6847816d981547207 100644
--- a/web/src/apps/AppRail.tsx
+++ b/web/src/apps/AppRail.tsx
@@ -13,6 +13,7 @@ interface RailApplication {
deepLink?: string;
templateKind?: string;
templateConnector?: string;
+ navPlacement?: { order?: number };
openTarget?: { database?: string };
compatibleDatabases?: string[];
capabilities?: Record;
@@ -31,7 +32,8 @@ function railApplications(body: unknown): RailApplication[] {
|| row.capabilities?.read === false) return false;
seen.add(row.id);
return true;
- }).sort((a, b) => a.name.localeCompare(b.name, undefined, { sensitivity: "base" }) || a.id.localeCompare(b.id));
+ }).sort((a, b) => (a.navPlacement?.order ?? 0) - (b.navPlacement?.order ?? 0)
+ || a.name.localeCompare(b.name, undefined, { sensitivity: "base" }) || a.id.localeCompare(b.id));
}
/** An Amazon App's rail row draws its own mark, never the Amazon logo (QI37). */
diff --git a/web/src/apps/account-health/AppView.tsx b/web/src/apps/account-health/AppView.tsx
index 72d50fd6969c8f91440ceec624a242f49c83043e..2249c123dff1fe8b9465188230e9442cecead53f 100644
--- a/web/src/apps/account-health/AppView.tsx
+++ b/web/src/apps/account-health/AppView.tsx
@@ -1,5 +1,6 @@
-import { useCallback, useEffect, useRef, useState } from "react";
+import { useCallback, useEffect, useId, useRef, useState } from "react";
import { API_V1, CREDENTIALS } from "../../apiContract";
+import { BodyPortal, useOverlayLayer } from "../../customer-grid/OverlaySurface";
import { useEscapeClose } from "../../ui/useEscapeClose";
import { AppTabs } from "../AppTabs";
import { LineChart } from "../../viz/kit";
@@ -7,7 +8,7 @@ import Dashboard from "./Dashboard";
import "../../styles/sdlc79/account-health.css";
import "../../styles/sdlc81/account-health-risk.css";
-// The dispatcher owns the App frame. This view owns its lists and right-side detail panel.
+// The dispatcher owns the App frame. This view owns its lists and detail overlays.
/** C2: the App as the dispatcher passes it. Structurally equal to lane A's AppViewProps application. */
interface AccountHealthApplication {
@@ -93,16 +94,30 @@ interface RiskPage {
interface Point { day: string; value: number | null; target: number | null; status: string }
interface HistoryPage { points: Point[]; cursor: string | null; label: string; unit: string }
+interface Category {
+ key: string; label: string; count: number | null; status: string; target: string;
+ window: { from: string; to: string }; sourceNote: string; sellerCentralUrl: string;
+ related: { asin: string; sku: string; issue: string; severity: string }[];
+}
+type CategoryLoad = Load | { phase: "refused"; message: string };
type Load = { phase: "loading"; value?: T } | { phase: "error"; message: string; value?: T }
| { phase: "ready"; value: T };
-type Picked = { kind: "metric" | "alert" | "risk"; id: string; metric?: Metric; alert?: Alert; risk?: Risk; linked?: boolean };
+type Picked = { kind: "metric" | "category" | "alert" | "risk"; id: string; metric?: Metric; alert?: Alert; risk?: Risk; linked?: boolean };
type Tab = "dashboard" | "metrics" | "alerts" | "risks";
const EMPTY = "No account health data";
const ALERT_STATUSES = ["Open", "Investigating", "Resolved", "Dismissed"];
const PAGE = 50;
const HISTORY_PAGE = 50;
+const CATEGORY_KEYS = new Set([
+ "listing_policy_violations", "product_authenticity_customer_complaints", "product_condition_customer_complaints",
+ "product_safety_customer_complaints", "food_and_product_safety_issues", "received_intellectual_property_complaints",
+ "suspected_intellectual_property_violations", "restricted_product_policy_violations",
+ "customer_product_reviews_policy_violations", "other_policy_violations", "document_requests",
+]);
+const categoryKey = (row: Metric) => row.key || row.id.split("@")[0];
+const isCategory = (row: Metric) => CATEGORY_KEYS.has(categoryKey(row));
const text = (value: unknown) => (typeof value === "string" ? value : "");
const num = (value: unknown) => (typeof value === "number" && Number.isFinite(value) ? value : null);
@@ -233,6 +248,21 @@ function parseHistory(raw: unknown): HistoryPage {
};
}
+function parseCategory(raw: unknown, key: string): Category {
+ const body = record(raw);
+ if (text(body.key) !== key || body.recordList !== false) throw new Error("This category could not be read.");
+ const window = record(body.window);
+ return {
+ key, label: text(body.label), count: num(body.count), status: text(body.status) || "No data",
+ target: text(body.target), window: { from: text(window.from), to: text(window.to) },
+ sourceNote: text(body.sourceNote), sellerCentralUrl: text(body.sellerCentralUrl),
+ related: list(body.related).map((raw) => {
+ const row = record(raw);
+ return { asin: text(row.asin), sku: text(row.sku), issue: text(row.issue), severity: text(row.severity) };
+ }),
+ };
+}
+
/** "Oct 22" from an ISO day. A calendar day is read in UTC. */
function day(iso: string) {
if (!iso) return "";
@@ -294,6 +324,135 @@ function Failed({ message, onRetry }: { message: string; onRetry: () => void })
;
}
+function CategoryPopup({ metric, appId, onClose }: { metric: Metric; appId: string; onClose: () => void }) {
+ const panel = useRef(null);
+ const titleId = useId();
+ const noteId = useId();
+ const [load, setLoad] = useState({ phase: "loading" });
+ const [attempt, setAttempt] = useState(0);
+ useOverlayLayer({ panelRef: panel, onDismiss: onClose, dismissOnOutside: false,
+ initialFocus: "[data-category-close]", restoreFocus: true, trapFocus: true });
+
+ useEffect(() => {
+ const controller = new AbortController();
+ setLoad({ phase: "loading" });
+ const query = new URLSearchParams({ app: appId });
+ void fetch(`${API_V1}/account-health/categories/${encodeURIComponent(metric.id)}?${query.toString()}`,
+ { credentials: CREDENTIALS, signal: controller.signal })
+ .then(async (response) => {
+ if (response.status === 401 || response.status === 403) {
+ const message = await readJson(response, "You do not have access to this category.")
+ .then(() => "You do not have access to this category.")
+ .catch((reason: unknown) => failure(reason, "You do not have access to this category."));
+ if (!controller.signal.aborted) setLoad({ phase: "refused", message });
+ return;
+ }
+ const category = parseCategory(await readJson(response, "This category could not be read."), categoryKey(metric));
+ if (!controller.signal.aborted) setLoad({ phase: "ready", value: category });
+ })
+ .catch((reason: unknown) => {
+ if (!controller.signal.aborted) setLoad({ phase: "error", message: failure(reason, "This category could not be read.") });
+ });
+ return () => controller.abort();
+ }, [appId, metric, attempt]);
+
+ const category = load.phase === "ready" ? load.value : null;
+ const relatedIncomplete = Boolean(category
+ && /unavailable|need access|missing or unreadable|missing from the saved report/i.test(category.sourceNote));
+ const count = category ? amount(category.count, "count") : "";
+ const dataState = load.phase !== "ready" ? load.phase === "refused" ? "no-permission" : load.phase
+ : category?.count === null ? "na" : category?.count === 0 ? "empty" : "ready";
+
+ return
+
{inaccessible ? ACCESS_REFUSED_MESSAGE : notBuilt ? "This table is not built yet. An admin Refresh builds it."
- : "Every SKU has its data."}
+
+ {loading ?
+ :
{inaccessible ? ACCESS_REFUSED_MESSAGE : probeFailed ? "This table could not be checked. Switch tabs to try again."
+ : notBuilt ? "This table is not built yet. An admin Refresh builds it." : "Every SKU has its data."}
- {result.state === "waiting_for_completion" ? "Complete the project to view results"
- : result.state === "waiting_for_window" || result.state === "waiting_for_completed_day"
- ? "Results appear when this period ends" : "Results unavailable"}
-
;
- const values = [...current.series, ...comparison.series].map(point => point.value)
- .filter((value): value is number => value !== null && Number.isFinite(value));
- if (!values.length) return