Deploy AIOS web (React glide grid + FastAPI slice)
Browse files- Dockerfile +5 -5
- RELEASES.json +1 -1
- VERSION +1 -1
- api/amazon_relational.py +24 -0
- api/connectors_amazon.py +6 -3
- api/deps.py +6 -25
- api/routes_admin.py +8 -6
- api/routes_auth.py +74 -93
- api/routes_keychain.py +7 -5
- api/routes_sheet.py +3 -2
- platform/core/auth_google.py +2 -2
- platform/core/sheets.py +92 -6
- platform/core/users.py +170 -49
- platform/harness/amazon_client.py +216 -23
- platform/harness/amazon_store.py +539 -17
- requirements.txt +3 -2
Dockerfile
CHANGED
|
@@ -20,12 +20,12 @@
|
|
| 20 |
# ---- stage 1: build the React bundle ----
|
| 21 |
FROM node:22-slim AS web
|
| 22 |
WORKDIR /web
|
| 23 |
-
#
|
| 24 |
-
#
|
| 25 |
-
COPY web/package.json ./
|
| 26 |
-
RUN npm
|
| 27 |
COPY web/ ./
|
| 28 |
-
RUN
|
| 29 |
|
| 30 |
# ---- stage 2: python API serving the bundle ----
|
| 31 |
FROM python:3.12-slim
|
|
|
|
| 20 |
# ---- stage 1: build the React bundle ----
|
| 21 |
FROM node:22-slim AS web
|
| 22 |
WORKDIR /web
|
| 23 |
+
# The lockfile is the audited supply-chain boundary. It now resolves entirely from the public
|
| 24 |
+
# registry; discarding it here made every image an unreviewed dependency upgrade.
|
| 25 |
+
COPY web/package.json web/package-lock.json ./
|
| 26 |
+
RUN npm ci --no-audit --no-fund --ignore-scripts
|
| 27 |
COPY web/ ./
|
| 28 |
+
RUN npm run build # -> /web/dist
|
| 29 |
|
| 30 |
# ---- stage 2: python API serving the bundle ----
|
| 31 |
FROM python:3.12-slim
|
RELEASES.json
CHANGED
|
@@ -1,5 +1,5 @@
|
|
| 1 |
{
|
| 2 |
-
"current": "
|
| 3 |
"releases": [
|
| 4 |
{
|
| 5 |
"version": "v71",
|
|
|
|
| 1 |
{
|
| 2 |
+
"current": "bcd2ba1",
|
| 3 |
"releases": [
|
| 4 |
{
|
| 5 |
"version": "v71",
|
VERSION
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
|
|
|
|
| 1 |
+
bcd2ba1
|
api/amazon_relational.py
CHANGED
|
@@ -95,6 +95,27 @@ IDENTITY = {k: tuple(_SPEC[k].identity) for k in SOURCE if k in _SPEC}
|
|
| 95 |
LINKS = {k: [(c, t, o) for c, t, o, _cardinality in _SPEC[k].links]
|
| 96 |
for k in SOURCE if k in _SPEC and _SPEC[k].links}
|
| 97 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 98 |
#: Columns whose NAME makes them a date whatever they are stored as. Amazon reports ISO-8601.
|
| 99 |
_DATEISH = ("purchase_date", "last_updated_date", "payments_date", "settlement_start_date",
|
| 100 |
"settlement_end_date", "deposit_date", "posted_date", "open_date", "date",
|
|
@@ -315,6 +336,9 @@ def apply_plan(rt, built, username="amazon", today=None):
|
|
| 315 |
for key, label, fields, rows in plans:
|
| 316 |
written[key] = _odoo._ensure_table_inplace(doc, key, label, fields, rows,
|
| 317 |
username, stamp)
|
|
|
|
|
|
|
|
|
|
| 318 |
return doc
|
| 319 |
|
| 320 |
rt.update(_odoo._ut().STORE_KEY, _apply_all, flush="sync")
|
|
|
|
| 95 |
LINKS = {k: [(c, t, o) for c, t, o, _cardinality in _SPEC[k].links]
|
| 96 |
for k in SOURCE if k in _SPEC and _SPEC[k].links}
|
| 97 |
|
| 98 |
+
|
| 99 |
+
def source_attribution(key):
|
| 100 |
+
"""Durable origin metadata for one materialised Amazon database.
|
| 101 |
+
|
| 102 |
+
DPP 1.8 permits a database separation strategy or a mechanism that tags and identifies the
|
| 103 |
+
origin of data. The DuckDB mirror uses the reserved ``amzn_*`` namespace already; this stamp
|
| 104 |
+
is the corresponding mechanism in the shared ``user_tables`` document. It is written on
|
| 105 |
+
every refresh so a user-facing label change cannot erase the Amazon origin or the exact raw
|
| 106 |
+
table/report that supplied the rows.
|
| 107 |
+
"""
|
| 108 |
+
if key not in SOURCE or key not in _SPEC:
|
| 109 |
+
raise KeyError(f"unknown Amazon database {key!r}")
|
| 110 |
+
return {
|
| 111 |
+
"key": "amazon_sp",
|
| 112 |
+
"origin": "Amazon Services API",
|
| 113 |
+
"rawTable": SOURCE[key],
|
| 114 |
+
"native": _SPEC[key].native,
|
| 115 |
+
"readOnly": True,
|
| 116 |
+
}
|
| 117 |
+
|
| 118 |
+
|
| 119 |
#: Columns whose NAME makes them a date whatever they are stored as. Amazon reports ISO-8601.
|
| 120 |
_DATEISH = ("purchase_date", "last_updated_date", "payments_date", "settlement_start_date",
|
| 121 |
"settlement_end_date", "deposit_date", "posted_date", "open_date", "date",
|
|
|
|
| 336 |
for key, label, fields, rows in plans:
|
| 337 |
written[key] = _odoo._ensure_table_inplace(doc, key, label, fields, rows,
|
| 338 |
username, stamp)
|
| 339 |
+
# DPP 1.8 attribution is explicit in the stored definition, not inferred from a
|
| 340 |
+
# mutable display label or from the process having imported this module.
|
| 341 |
+
doc[key]["connector"] = source_attribution(key)
|
| 342 |
return doc
|
| 343 |
|
| 344 |
rt.update(_odoo._ut().STORE_KEY, _apply_all, flush="sync")
|
api/connectors_amazon.py
CHANGED
|
@@ -13,7 +13,7 @@ caller that imports it from here still works.
|
|
| 13 |
`keychain.register_prober` is the slot; `datastore.set_paused_probe` is the same idiom one layer
|
| 14 |
down.
|
| 15 |
|
| 16 |
-
β **READ-ONLY.** See the client's header for the promise and
|
| 17 |
"""
|
| 18 |
from __future__ import annotations
|
| 19 |
|
|
@@ -29,11 +29,14 @@ if str(_PLATFORM) not in sys.path:
|
|
| 29 |
# β RE-EXPORTED, NOT REDEFINED. One definition of the credential shape, the endpoints, the report
|
| 30 |
# map and the transport; this module adds the probe and the registration and nothing else.
|
| 31 |
from harness.amazon_client import ( # noqa: F401,E402
|
|
|
|
| 32 |
CREDENTIAL_FIELDS, DEFAULT_REGION, ENDPOINTS, KEY, LABEL, LWA_TOKEN_URL, OPTIONAL_FIELDS,
|
| 33 |
REPORT_TYPES, SANDBOX_ENDPOINTS, base_for, is_sandbox,
|
| 34 |
ROLES_REFUSED, ROLES_TAKEN, US_MARKETPLACE_ID, AmazonError, _create_report,
|
| 35 |
-
|
| 36 |
-
|
|
|
|
|
|
|
| 37 |
)
|
| 38 |
|
| 39 |
#: β MUST MATCH `core.keychain.ENTRY_TYPES` OR `add_entry` 400s with a message about our data
|
|
|
|
| 13 |
`keychain.register_prober` is the slot; `datastore.set_paused_probe` is the same idiom one layer
|
| 14 |
down.
|
| 15 |
|
| 16 |
+
β **READ-ONLY.** See the client's header for the promise and its two fixed POST wrappers.
|
| 17 |
"""
|
| 18 |
from __future__ import annotations
|
| 19 |
|
|
|
|
| 29 |
# β RE-EXPORTED, NOT REDEFINED. One definition of the credential shape, the endpoints, the report
|
| 30 |
# map and the transport; this module adds the probe and the registration and nothing else.
|
| 31 |
from harness.amazon_client import ( # noqa: F401,E402
|
| 32 |
+
AWD_INVENTORY_PAGE_SIZE, COMPETITIVE_SUMMARY_BATCH_SIZE,
|
| 33 |
CREDENTIAL_FIELDS, DEFAULT_REGION, ENDPOINTS, KEY, LABEL, LWA_TOKEN_URL, OPTIONAL_FIELDS,
|
| 34 |
REPORT_TYPES, SANDBOX_ENDPOINTS, base_for, is_sandbox,
|
| 35 |
ROLES_REFUSED, ROLES_TAKEN, US_MARKETPLACE_ID, AmazonError, _create_report,
|
| 36 |
+
_competitive_summary_batch, _COMPETITIVE_SUMMARY_BATCH_PATH,
|
| 37 |
+
_COMPETITIVE_SUMMARY_ITEM_URI, _download_document, _post_form, _REPORTS_CREATE_PATH,
|
| 38 |
+
_request, _safe, _TIMEOUT, _AWD_INVENTORY_PATH, access_token, awd_inventory,
|
| 39 |
+
competitive_summaries, marketplace_participations, report_document,
|
| 40 |
)
|
| 41 |
|
| 42 |
#: β MUST MATCH `core.keychain.ENTRY_TYPES` OR `add_entry` 400s with a message about our data
|
api/deps.py
CHANGED
|
@@ -776,24 +776,16 @@ def _user_for(claims):
|
|
| 776 |
"""The account named by a verified cookie, or None β with the epoch check that makes a
|
| 777 |
stateless cookie revocable.
|
| 778 |
|
| 779 |
-
β THE RULE THIS FUNCTION MUST OBEY: **it has to accept exactly the identities
|
| 780 |
issues.** Any divergence produces the worst failure shape there is β a login that returns 200
|
| 781 |
-
with a cookie and then 401s every request after it.
|
| 782 |
-
emergency-master branch below ran only when NO record existed, while `verify` grants the master
|
| 783 |
-
identity even when one does), so it is now written as a deliberate mirror of `verify`'s
|
| 784 |
-
structure: try the record, else the master.
|
| 785 |
|
| 786 |
-
TWO CASES
|
| 787 |
* THE RECORD, if it is readable and satisfies everything a session adds on top of a login:
|
| 788 |
the account is active, and its CURRENT epoch equals the cookie's β otherwise the session
|
| 789 |
is revoked (a password change or a deactivation bumped it).
|
| 790 |
-
* THE
|
| 791 |
-
|
| 792 |
-
store outage, a self-deactivation or a bumped epoch locking the owner out of the product.
|
| 793 |
-
β AND IT MEANS AN `admin` SESSION IS NOT EPOCH-REVOCABLE while APP_PASSWORD is set. That
|
| 794 |
-
is not a weakening: whoever holds APP_PASSWORD can simply log in again, so bumping admin's
|
| 795 |
-
epoch never revoked them in the first place. Rotating APP_PASSWORD is how you revoke it.
|
| 796 |
-
Every OTHER account is fully epoch-revocable, which is asserted in `verify_api.py`.
|
| 797 |
* anything else β None. An unknown username is NOT admitted just because its signature was
|
| 798 |
valid: a signature proves the cookie is ours, not that the account still exists.
|
| 799 |
"""
|
|
@@ -831,17 +823,6 @@ def _user_for(claims):
|
|
| 831 |
if claim_tenant != pub.get("tenant"):
|
| 832 |
return None
|
| 833 |
return pub
|
| 834 |
-
if (uname == "admin" and os.environ.get("APP_PASSWORD", "")
|
| 835 |
-
and claim_tenant == "royal-imports"):
|
| 836 |
-
# The emergency master is TENANT #0's break-glass, not the platform's: a synthetic
|
| 837 |
-
# all-access identity minted into another tenant's session would be the exact
|
| 838 |
-
# cross-tenant widening the rule above closes.
|
| 839 |
-
# β WAVE 40 (R2): the same mark `users.verify`'s master branch carries. The two dicts are
|
| 840 |
-
# documented mirrors of each other, so the key rides on both or they are not mirrors.
|
| 841 |
-
return {"username": "admin", "name": "Administrator", "role": "admin",
|
| 842 |
-
"bus": "all", "modules": "all", "tenant": "royal-imports",
|
| 843 |
-
"emergency_master": True,
|
| 844 |
-
"epoch": int(claims.get("e") or 0)}
|
| 845 |
return None
|
| 846 |
|
| 847 |
|
|
@@ -862,7 +843,7 @@ def _user_for(claims):
|
|
| 862 |
# HONEST LIMITS, stated rather than discovered later: the map is per-process, so N workers can each
|
| 863 |
# write once per window (still bounded); a restart re-arms every user's first request; and the
|
| 864 |
# resolution of `last_active` is therefore "within the last hour", which is exactly what the plane
|
| 865 |
-
# renders it as. Bounded
|
| 866 |
# with many accounts.
|
| 867 |
_ACTIVE_SEEN: "OrderedDict[str, float]" = OrderedDict()
|
| 868 |
_ACTIVE_EVERY = float(os.environ.get("AIOS_ACTIVE_STAMP_SECONDS") or 3600)
|
|
|
|
| 776 |
"""The account named by a verified cookie, or None β with the epoch check that makes a
|
| 777 |
stateless cookie revocable.
|
| 778 |
|
| 779 |
+
β THE RULE THIS FUNCTION MUST OBEY: **it has to accept exactly the identities the login route
|
| 780 |
issues.** Any divergence produces the worst failure shape there is β a login that returns 200
|
| 781 |
+
with a cookie and then 401s every request after it.
|
|
|
|
|
|
|
|
|
|
| 782 |
|
| 783 |
+
TWO CASES:
|
| 784 |
* THE RECORD, if it is readable and satisfies everything a session adds on top of a login:
|
| 785 |
the account is active, and its CURRENT epoch equals the cookie's β otherwise the session
|
| 786 |
is revoked (a password change or a deactivation bumped it).
|
| 787 |
+
* THE QA RUNNER, only when both non-production deployment gates remain enabled and its
|
| 788 |
+
password-derived epoch still matches. Live deployments cannot enable it.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 789 |
* anything else β None. An unknown username is NOT admitted just because its signature was
|
| 790 |
valid: a signature proves the cookie is ours, not that the account still exists.
|
| 791 |
"""
|
|
|
|
| 823 |
if claim_tenant != pub.get("tenant"):
|
| 824 |
return None
|
| 825 |
return pub
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 826 |
return None
|
| 827 |
|
| 828 |
|
|
|
|
| 843 |
# HONEST LIMITS, stated rather than discovered later: the map is per-process, so N workers can each
|
| 844 |
# write once per window (still bounded); a restart re-arms every user's first request; and the
|
| 845 |
# resolution of `last_active` is therefore "within the last hour", which is exactly what the plane
|
| 846 |
+
# renders it as. Bounded independently so it cannot grow into a leak on a deployment
|
| 847 |
# with many accounts.
|
| 848 |
_ACTIVE_SEEN: "OrderedDict[str, float]" = OrderedDict()
|
| 849 |
_ACTIVE_EVERY = float(os.environ.get("AIOS_ACTIVE_STAMP_SECONDS") or 3600)
|
api/routes_admin.py
CHANGED
|
@@ -216,10 +216,12 @@ def _clean_username(v):
|
|
| 216 |
return uname
|
| 217 |
|
| 218 |
|
| 219 |
-
def _clean_password(v, username=None, name=None, email=None, record=None
|
|
|
|
| 220 |
pw = str(v or "")
|
| 221 |
failures = users.password_failures(pw, username=username, name=name, email=email,
|
| 222 |
-
record=record
|
|
|
|
| 223 |
if failures:
|
| 224 |
# Name every broken rule so the caller can correct the value without guessing the policy.
|
| 225 |
raise err(400, "weak_password",
|
|
@@ -414,9 +416,8 @@ def update_user(username: str, body: dict = Body(default=None),
|
|
| 414 |
is_self = uname == session.uname
|
| 415 |
role = _clean_role(body["role"]) if "role" in body else None
|
| 416 |
active = bool(body["active"]) if "active" in body else None
|
| 417 |
-
# THE ONE-CLICK LOCKOUT GUARD.
|
| 418 |
-
#
|
| 419 |
-
# their own access with a toggle and having to go find the master password.
|
| 420 |
if is_self and role is not None and role != "admin":
|
| 421 |
raise err(400, "self_demote",
|
| 422 |
"you cannot remove your own administrator role; ask another admin")
|
|
@@ -467,7 +468,8 @@ def set_password(username: str, body: dict = Body(default=None),
|
|
| 467 |
# 404, never 403, because "that account exists in another company" is itself a leak.
|
| 468 |
raise err(404, "no_such_user", f"no account named {uname!r}")
|
| 469 |
pw = _clean_password((body or {}).get("password"), username=uname,
|
| 470 |
-
name=reg[uname].get("name"), email=reg[uname].get("email"),
|
|
|
|
| 471 |
before = int(reg[uname].get("epoch") or 0)
|
| 472 |
try:
|
| 473 |
# This route just checked strength AND reuse against its record. Spend the history's
|
|
|
|
| 216 |
return uname
|
| 217 |
|
| 218 |
|
| 219 |
+
def _clean_password(v, username=None, name=None, email=None, record=None,
|
| 220 |
+
allow_early_rotation=False):
|
| 221 |
pw = str(v or "")
|
| 222 |
failures = users.password_failures(pw, username=username, name=name, email=email,
|
| 223 |
+
record=record,
|
| 224 |
+
allow_early_rotation=allow_early_rotation)
|
| 225 |
if failures:
|
| 226 |
# Name every broken rule so the caller can correct the value without guessing the policy.
|
| 227 |
raise err(400, "weak_password",
|
|
|
|
| 416 |
is_self = uname == session.uname
|
| 417 |
role = _clean_role(body["role"]) if "role" in body else None
|
| 418 |
active = bool(body["active"]) if "active" in body else None
|
| 419 |
+
# THE ONE-CLICK LOCKOUT GUARD. There is deliberately no APP_PASSWORD bypass, so preventing a
|
| 420 |
+
# stored administrator from removing their own access is part of keeping recovery possible.
|
|
|
|
| 421 |
if is_self and role is not None and role != "admin":
|
| 422 |
raise err(400, "self_demote",
|
| 423 |
"you cannot remove your own administrator role; ask another admin")
|
|
|
|
| 468 |
# 404, never 403, because "that account exists in another company" is itself a leak.
|
| 469 |
raise err(404, "no_such_user", f"no account named {uname!r}")
|
| 470 |
pw = _clean_password((body or {}).get("password"), username=uname,
|
| 471 |
+
name=reg[uname].get("name"), email=reg[uname].get("email"),
|
| 472 |
+
record=reg[uname], allow_early_rotation=True)
|
| 473 |
before = int(reg[uname].get("epoch") or 0)
|
| 474 |
try:
|
| 475 |
# This route just checked strength AND reuse against its record. Spend the history's
|
api/routes_auth.py
CHANGED
|
@@ -7,11 +7,7 @@ replaces, and it is deliberately the same account store so the migration is a sw
|
|
| 7 |
CREDENTIAL check, not of the user model.
|
| 8 |
"""
|
| 9 |
import base64 as _b64
|
| 10 |
-
import hmac
|
| 11 |
-
import os
|
| 12 |
import re as _re
|
| 13 |
-
import time
|
| 14 |
-
from collections import OrderedDict
|
| 15 |
|
| 16 |
from fastapi import APIRouter, Body, Depends, Request, Response
|
| 17 |
|
|
@@ -20,49 +16,46 @@ from deps import Session, err, require_session, users, perms
|
|
| 20 |
|
| 21 |
router = APIRouter(prefix="/api/v1/auth")
|
| 22 |
|
| 23 |
-
#: THE SERVER OWNS BACKOFF
|
| 24 |
-
#:
|
| 25 |
-
#:
|
| 26 |
-
#:
|
| 27 |
-
|
| 28 |
-
|
| 29 |
-
|
| 30 |
-
|
| 31 |
-
|
| 32 |
-
|
| 33 |
-
_FAILS = OrderedDict()
|
| 34 |
-
_MAX_TRACKED = 2048
|
| 35 |
-
_LOCKOUT_AFTER = 8
|
| 36 |
-
_LOCKOUT_SECONDS = 60
|
| 37 |
|
| 38 |
|
| 39 |
def _throttled(key, now=None):
|
| 40 |
-
|
| 41 |
-
hit = _FAILS.get(key)
|
| 42 |
-
if not hit:
|
| 43 |
-
return 0
|
| 44 |
-
count, last = hit
|
| 45 |
-
if count < _LOCKOUT_AFTER:
|
| 46 |
return 0
|
| 47 |
-
|
| 48 |
-
|
|
|
|
|
|
|
|
|
|
| 49 |
|
| 50 |
|
| 51 |
def _note_failure(key, now=None):
|
| 52 |
-
|
| 53 |
-
|
| 54 |
-
|
| 55 |
-
|
| 56 |
-
|
| 57 |
-
|
| 58 |
-
|
| 59 |
-
_FAILS.move_to_end(key)
|
| 60 |
-
while len(_FAILS) > _MAX_TRACKED:
|
| 61 |
-
_FAILS.popitem(last=False)
|
| 62 |
|
| 63 |
|
| 64 |
def _clear_failures(key):
|
| 65 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 66 |
|
| 67 |
|
| 68 |
def _public_user(user):
|
|
@@ -145,19 +138,18 @@ def login(request: Request, response: Response, body: dict = Body(default=None))
|
|
| 145 |
pw = str(body.get("password") or "")
|
| 146 |
code = str(body.get("mfaCode") or "").strip()
|
| 147 |
|
| 148 |
-
# Throttle on the IDENTIFIER alone: the tenant is not known until the credential resolves,
|
| 149 |
-
# and a per-(tenant, uname) key would let an attacker reset their budget by rotating slugs.
|
| 150 |
-
throttle_key = ("*", uname)
|
| 151 |
-
wait = _throttled(throttle_key)
|
| 152 |
-
if wait:
|
| 153 |
-
raise err(429, "too_many_attempts",
|
| 154 |
-
f"too many failed attempts; try again in {wait} seconds")
|
| 155 |
-
|
| 156 |
# W39-T28 / A62. Resolve this synthetic identity FIRST. The normal account registry is
|
| 157 |
# tenant #0's document on the PostgreSQL path, so even a harmless `verify()` read would make
|
| 158 |
# a qa-b proof touch the wrong schema. `qa_identity` is inert unless both staging flags and
|
| 159 |
# APP_PASSWORD are present; every ordinary username falls straight through to `verify()`.
|
| 160 |
user = users.qa_identity(uname, pw) if (uname and pw) else None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 161 |
if not user:
|
| 162 |
user = users.verify(uname, pw) if (uname and pw) else None
|
| 163 |
if not user:
|
|
@@ -182,9 +174,9 @@ def login(request: Request, response: Response, body: dict = Body(default=None))
|
|
| 182 |
|
| 183 |
# W57-T35 / C12: a correct credential may still need rotation. Check BEFORE bumping or
|
| 184 |
# minting so an expired password gets the change door and no session, without counting a
|
| 185 |
-
# successful authentication against the guessing budget.
|
| 186 |
-
# stored password clock
|
| 187 |
-
if not
|
| 188 |
try:
|
| 189 |
rec = (users.registry() or {}).get(user["username"])
|
| 190 |
except Exception:
|
|
@@ -200,10 +192,8 @@ def login(request: Request, response: Response, body: dict = Body(default=None))
|
|
| 200 |
pass
|
| 201 |
|
| 202 |
# W57-T36 / C12: every STORED account needs its second factor before any cookie is minted.
|
| 203 |
-
#
|
| 204 |
-
|
| 205 |
-
# Neither has a stored factor this route can require without defeating its existing purpose.
|
| 206 |
-
if not (user.get("qa_runner") or user.get("emergency_master")):
|
| 207 |
try:
|
| 208 |
rec = (users.registry() or {}).get(user["username"])
|
| 209 |
except Exception:
|
|
@@ -233,21 +223,20 @@ def login(request: Request, response: Response, body: dict = Body(default=None))
|
|
| 233 |
# the bump; minting from it would issue every cookie in the product at the OLD epoch, which
|
| 234 |
# `_user_for` refuses on the very next request β "200, then 401 forever".
|
| 235 |
#
|
| 236 |
-
#
|
| 237 |
-
#
|
| 238 |
-
#
|
| 239 |
-
# NOTHING on tenant #0's `users` key, which a bump's read-modify-write would violate.
|
| 240 |
-
# * `emergency_master` β `deps._user_for` admits that identity whatever the record's epoch
|
| 241 |
-
# says, so a bump revokes nothing there and only risks a store write on the break-glass
|
| 242 |
-
# path that exists precisely for a store outage.
|
| 243 |
#
|
| 244 |
# A FAILED BUMP MUST NOT FAIL THE LOGIN. `store.update` raises `StoreWriteRefused` where the
|
| 245 |
# deployment may not write this tenant (mapped to 503 in `main.py`) and `RuntimeError` when the
|
| 246 |
# store is unavailable, so an unguarded bump would turn every sign-in on such a deployment into
|
| 247 |
# a 503. The sign-in stands at the epoch it can prove; only the sign-out-everywhere half is
|
| 248 |
# lost, and the log says so rather than leaving it silent.
|
|
|
|
|
|
|
|
|
|
| 249 |
session_epoch = int(user.get("epoch") or 0)
|
| 250 |
-
if not
|
| 251 |
try:
|
| 252 |
bumped = users.bump_epoch(user["username"])
|
| 253 |
if bumped is not None:
|
|
@@ -260,9 +249,8 @@ def login(request: Request, response: Response, body: dict = Body(default=None))
|
|
| 260 |
aios_session.set_cookie(response, request, value)
|
| 261 |
# Wave 19 (R4): the login stamp. THE RESOLVED username, never the posted identifier β the
|
| 262 |
# person may have typed an email address and `users.verify` resolved it to the registry key;
|
| 263 |
-
# stamping what was typed would write onto a key that does not exist.
|
| 264 |
-
#
|
| 265 |
-
# so the one thing this cannot do is turn a good credential into a failed sign-in.
|
| 266 |
# The synthetic QA identity has no normal account record to stamp. More importantly, a
|
| 267 |
# registry stamp here would default to tenant #0 and make a qa-b proof touch the wrong schema.
|
| 268 |
if not user.get("qa_runner"):
|
|
@@ -273,9 +261,6 @@ def login(request: Request, response: Response, body: dict = Body(default=None))
|
|
| 273 |
import deps as _deps
|
| 274 |
if not user.get("qa_runner"):
|
| 275 |
_deps.note_active(user["username"])
|
| 276 |
-
# MOVED from password success: only a WHOLE successful login clears the budget. Otherwise
|
| 277 |
-
# a known password would reset the counter before every wrong second-factor guess.
|
| 278 |
-
_clear_failures(throttle_key)
|
| 279 |
return {"user": _public_user(user)}
|
| 280 |
|
| 281 |
|
|
@@ -287,16 +272,16 @@ _MFA_ALREADY_ENROLLED = ("Two step sign in is already on for this account. Ask a
|
|
| 287 |
|
| 288 |
def _mfa_credentials(uname, password):
|
| 289 |
"""The MFA doors share login's credential check and guessing budget, including aliases."""
|
| 290 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 291 |
wait = _throttled(throttle_key)
|
| 292 |
if wait:
|
| 293 |
raise err(429, "too_many_attempts",
|
| 294 |
f"too many failed attempts. Try again in {wait} seconds")
|
| 295 |
-
|
| 296 |
-
# record exists to enrol. The emergency master is refused for the same no-stored-factor rule.
|
| 297 |
-
user = users.qa_identity(uname, password) if (uname and password) else None
|
| 298 |
-
if not user:
|
| 299 |
-
user = users.verify(uname, password) if (uname and password) else None
|
| 300 |
if not user:
|
| 301 |
try:
|
| 302 |
users._hash(password or "x", "00" * 16)
|
|
@@ -304,9 +289,7 @@ def _mfa_credentials(uname, password):
|
|
| 304 |
pass
|
| 305 |
_note_failure(throttle_key)
|
| 306 |
raise err(401, "invalid_credentials", "that username and password do not match")
|
| 307 |
-
|
| 308 |
-
raise err(400, "no_stored_account", "That sign in cannot enrol a second factor.")
|
| 309 |
-
return user
|
| 310 |
|
| 311 |
|
| 312 |
def _mfa_record(uname):
|
|
@@ -354,6 +337,8 @@ def _mfa_session(request, response, user, rec, tenant, landed, throttle_key):
|
|
| 354 |
if landed is None:
|
| 355 |
# A missing record/no-op write cannot support a cookie or a promise that codes were saved.
|
| 356 |
raise err(503, "store_unavailable", "the second factor was not saved. Try again")
|
|
|
|
|
|
|
| 357 |
value, _claims = aios_session.mint(tenant, user["username"], int(landed))
|
| 358 |
aios_session.set_cookie(response, request, value)
|
| 359 |
users.touch_login(user["username"])
|
|
@@ -362,7 +347,6 @@ def _mfa_session(request, response, user, rec, tenant, landed, throttle_key):
|
|
| 362 |
if not users.password_changed_at(rec):
|
| 363 |
# This is a successful first sign-in too: preserve T35's legacy clock backfill.
|
| 364 |
users.stamp_password_age(user["username"])
|
| 365 |
-
_clear_failures(throttle_key)
|
| 366 |
# Responses carrying a new secret or recovery codes must not survive in an HTTP cache.
|
| 367 |
response.headers["Cache-Control"] = "no-store"
|
| 368 |
|
|
@@ -376,7 +360,7 @@ def mfa_enroll(response: Response, body: dict = Body(default=None)):
|
|
| 376 |
"""
|
| 377 |
body = body or {}
|
| 378 |
uname = str(body.get("username") or "").strip().lower()
|
| 379 |
-
user = _mfa_credentials(uname, str(body.get("password") or ""))
|
| 380 |
rec = _mfa_record(user["username"])
|
| 381 |
if users.mfa_enrolled(rec):
|
| 382 |
raise err(409, "mfa_already_enrolled", _MFA_ALREADY_ENROLLED)
|
|
@@ -396,8 +380,7 @@ def mfa_enroll_confirm(request: Request, response: Response, body: dict = Body(d
|
|
| 396 |
"""
|
| 397 |
body = body or {}
|
| 398 |
uname = str(body.get("username") or "").strip().lower()
|
| 399 |
-
throttle_key = ("
|
| 400 |
-
user = _mfa_credentials(uname, str(body.get("password") or ""))
|
| 401 |
rec = _mfa_record(user["username"])
|
| 402 |
if users.mfa_enrolled(rec):
|
| 403 |
raise err(409, "mfa_already_enrolled", _MFA_ALREADY_ENROLLED)
|
|
@@ -416,8 +399,8 @@ def mfa_enroll_confirm(request: Request, response: Response, body: dict = Body(d
|
|
| 416 |
@router.get("/mfa")
|
| 417 |
def mfa_status(session: Session = Depends(require_session)):
|
| 418 |
"""The calling account's state only: never a secret, code or hash."""
|
| 419 |
-
#
|
| 420 |
-
if session.user.get("qa_runner")
|
| 421 |
rec = {}
|
| 422 |
else:
|
| 423 |
rec = _mfa_record(session.uname)
|
|
@@ -435,8 +418,7 @@ def mfa_recovery_codes(request: Request, response: Response, body: dict = Body(d
|
|
| 435 |
The clear codes appear only in this response; the store and all later reads hold only hashes.
|
| 436 |
"""
|
| 437 |
uname = session.uname
|
| 438 |
-
throttle_key = ("
|
| 439 |
-
user = _mfa_credentials(uname, str((body or {}).get("password") or ""))
|
| 440 |
# The session supplies the identity. Body username/tenant fields can never choose a target.
|
| 441 |
rec = _mfa_record(uname)
|
| 442 |
if not users.mfa_enrolled(rec):
|
|
@@ -452,6 +434,7 @@ def mfa_recovery_codes(request: Request, response: Response, body: dict = Body(d
|
|
| 452 |
def password_policy():
|
| 453 |
"""An account-independent checklist the sign-in page can show before authentication."""
|
| 454 |
return {"rules": users.password_rules(), "minLength": users.PASSWORD_MIN_LENGTH,
|
|
|
|
| 455 |
"maxAgeDays": users.PASSWORD_MAX_AGE_DAYS, "historyCount": users.PASSWORD_HISTORY}
|
| 456 |
|
| 457 |
|
|
@@ -462,18 +445,18 @@ def change_password(body: dict = Body(default=None)):
|
|
| 462 |
uname = str(body.get("username") or "").strip().lower()
|
| 463 |
pw = str(body.get("currentPassword") or "")
|
| 464 |
new_pw = str(body.get("newPassword") or "")
|
| 465 |
-
#
|
| 466 |
-
|
| 467 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 468 |
wait = _throttled(throttle_key)
|
| 469 |
if wait:
|
| 470 |
raise err(429, "too_many_attempts",
|
| 471 |
f"too many failed attempts. Try again in {wait} seconds")
|
| 472 |
-
|
| 473 |
-
# authenticates, but no stored password exists here, so reporting a rotation would be false.
|
| 474 |
-
user = users.qa_identity(uname, pw) if (uname and pw) else None
|
| 475 |
-
if not user:
|
| 476 |
-
user = users.verify(uname, pw) if (uname and pw) else None
|
| 477 |
if not user:
|
| 478 |
try:
|
| 479 |
users._hash(pw or "x", "00" * 16)
|
|
@@ -481,8 +464,6 @@ def change_password(body: dict = Body(default=None)):
|
|
| 481 |
pass
|
| 482 |
_note_failure(throttle_key)
|
| 483 |
raise err(401, "invalid_credentials", "that username and password do not match")
|
| 484 |
-
if user.get("emergency_master") or user.get("qa_runner"):
|
| 485 |
-
raise err(400, "no_stored_password", "This account has no stored password to change here.")
|
| 486 |
try:
|
| 487 |
# verify accepts email addresses too. Rotate the RESOLVED registry key, never the typed
|
| 488 |
# alias, or this door could return 204 without changing any password or revoking sessions.
|
|
|
|
| 7 |
CREDENTIAL check, not of the user model.
|
| 8 |
"""
|
| 9 |
import base64 as _b64
|
|
|
|
|
|
|
| 10 |
import re as _re
|
|
|
|
|
|
|
| 11 |
|
| 12 |
from fastapi import APIRouter, Body, Depends, Request, Response
|
| 13 |
|
|
|
|
| 16 |
|
| 17 |
router = APIRouter(prefix="/api/v1/auth")
|
| 18 |
|
| 19 |
+
#: THE SERVER OWNS BACKOFF: the login page never paces itself. The budget lives in the shared
|
| 20 |
+
#: tenant-0 auth store, not this process, so worker rotation or a deploy cannot buy ten more tries.
|
| 21 |
+
#: Username and email aliases resolve to one canonical account key; only a SHA-256 digest of that
|
| 22 |
+
#: normalised key is persisted. Any store failure fails the authentication door closed.
|
| 23 |
+
def _throttle_key(identifier):
|
| 24 |
+
try:
|
| 25 |
+
return users.canonical_login_identifier(identifier)
|
| 26 |
+
except Exception:
|
| 27 |
+
raise err(503, "auth_state_unavailable",
|
| 28 |
+
"sign in protection is temporarily unavailable. Try again later")
|
|
|
|
|
|
|
|
|
|
|
|
|
| 29 |
|
| 30 |
|
| 31 |
def _throttled(key, now=None):
|
| 32 |
+
if key is None: # staging-only synthetic QA identity
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 33 |
return 0
|
| 34 |
+
try:
|
| 35 |
+
return users.login_lockout_remaining(key, now=now)
|
| 36 |
+
except Exception:
|
| 37 |
+
raise err(503, "auth_state_unavailable",
|
| 38 |
+
"sign in protection is temporarily unavailable. Try again later")
|
| 39 |
|
| 40 |
|
| 41 |
def _note_failure(key, now=None):
|
| 42 |
+
if key is None:
|
| 43 |
+
return 0
|
| 44 |
+
try:
|
| 45 |
+
return users.note_login_failure(key, now=now)
|
| 46 |
+
except Exception:
|
| 47 |
+
raise err(503, "auth_state_unavailable",
|
| 48 |
+
"sign in protection is temporarily unavailable. Try again later")
|
|
|
|
|
|
|
|
|
|
| 49 |
|
| 50 |
|
| 51 |
def _clear_failures(key):
|
| 52 |
+
if key is None:
|
| 53 |
+
return
|
| 54 |
+
try:
|
| 55 |
+
users.clear_login_failures(key)
|
| 56 |
+
except Exception:
|
| 57 |
+
raise err(503, "auth_state_unavailable",
|
| 58 |
+
"sign in protection is temporarily unavailable. Try again later")
|
| 59 |
|
| 60 |
|
| 61 |
def _public_user(user):
|
|
|
|
| 138 |
pw = str(body.get("password") or "")
|
| 139 |
code = str(body.get("mfaCode") or "").strip()
|
| 140 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 141 |
# W39-T28 / A62. Resolve this synthetic identity FIRST. The normal account registry is
|
| 142 |
# tenant #0's document on the PostgreSQL path, so even a harmless `verify()` read would make
|
| 143 |
# a qa-b proof touch the wrong schema. `qa_identity` is inert unless both staging flags and
|
| 144 |
# APP_PASSWORD are present; every ordinary username falls straight through to `verify()`.
|
| 145 |
user = users.qa_identity(uname, pw) if (uname and pw) else None
|
| 146 |
+
# A correctly proved QA runner is a staging fixture and remains zero-touch against tenant #0.
|
| 147 |
+
# Every other attempt resolves username/email aliases to one durable account budget.
|
| 148 |
+
throttle_key = None if user else _throttle_key(uname)
|
| 149 |
+
wait = _throttled(throttle_key)
|
| 150 |
+
if wait:
|
| 151 |
+
raise err(429, "too_many_attempts",
|
| 152 |
+
f"too many failed attempts β try again in {wait} seconds")
|
| 153 |
if not user:
|
| 154 |
user = users.verify(uname, pw) if (uname and pw) else None
|
| 155 |
if not user:
|
|
|
|
| 174 |
|
| 175 |
# W57-T35 / C12: a correct credential may still need rotation. Check BEFORE bumping or
|
| 176 |
# minting so an expired password gets the change door and no session, without counting a
|
| 177 |
+
# successful authentication against the guessing budget. The staging QA fixture has no
|
| 178 |
+
# stored password clock; every production identity does.
|
| 179 |
+
if not user.get("qa_runner"):
|
| 180 |
try:
|
| 181 |
rec = (users.registry() or {}).get(user["username"])
|
| 182 |
except Exception:
|
|
|
|
| 192 |
pass
|
| 193 |
|
| 194 |
# W57-T36 / C12: every STORED account needs its second factor before any cookie is minted.
|
| 195 |
+
# qa_runner is the sole exception and is double-gated out of live deployments.
|
| 196 |
+
if not user.get("qa_runner"):
|
|
|
|
|
|
|
| 197 |
try:
|
| 198 |
rec = (users.registry() or {}).get(user["username"])
|
| 199 |
except Exception:
|
|
|
|
| 223 |
# the bump; minting from it would issue every cookie in the product at the OLD epoch, which
|
| 224 |
# `_user_for` refuses on the very next request β "200, then 401 forever".
|
| 225 |
#
|
| 226 |
+
# qa_runner is the sole exception: it has no registry record to bump (its epoch is derived
|
| 227 |
+
# from APP_PASSWORD), and `verify_api.py` asserts its login path reads and writes NOTHING on
|
| 228 |
+
# tenant #0's `users` key. Live deployments cannot enable this double-gated fixture.
|
|
|
|
|
|
|
|
|
|
|
|
|
| 229 |
#
|
| 230 |
# A FAILED BUMP MUST NOT FAIL THE LOGIN. `store.update` raises `StoreWriteRefused` where the
|
| 231 |
# deployment may not write this tenant (mapped to 503 in `main.py`) and `RuntimeError` when the
|
| 232 |
# store is unavailable, so an unguarded bump would turn every sign-in on such a deployment into
|
| 233 |
# a 503. The sign-in stands at the epoch it can prove; only the sign-out-everywhere half is
|
| 234 |
# lost, and the log says so rather than leaving it silent.
|
| 235 |
+
# A complete password+factor proof clears the durable budget BEFORE a cookie is minted. If
|
| 236 |
+
# that security write is unavailable, authentication fails closed.
|
| 237 |
+
_clear_failures(throttle_key)
|
| 238 |
session_epoch = int(user.get("epoch") or 0)
|
| 239 |
+
if not user.get("qa_runner"):
|
| 240 |
try:
|
| 241 |
bumped = users.bump_epoch(user["username"])
|
| 242 |
if bumped is not None:
|
|
|
|
| 249 |
aios_session.set_cookie(response, request, value)
|
| 250 |
# Wave 19 (R4): the login stamp. THE RESOLVED username, never the posted identifier β the
|
| 251 |
# person may have typed an email address and `users.verify` resolved it to the registry key;
|
| 252 |
+
# stamping what was typed would write onto a key that does not exist. `touch_login` is
|
| 253 |
+
# fail-silent, so it cannot turn a good credential into a failed sign-in.
|
|
|
|
| 254 |
# The synthetic QA identity has no normal account record to stamp. More importantly, a
|
| 255 |
# registry stamp here would default to tenant #0 and make a qa-b proof touch the wrong schema.
|
| 256 |
if not user.get("qa_runner"):
|
|
|
|
| 261 |
import deps as _deps
|
| 262 |
if not user.get("qa_runner"):
|
| 263 |
_deps.note_active(user["username"])
|
|
|
|
|
|
|
|
|
|
| 264 |
return {"user": _public_user(user)}
|
| 265 |
|
| 266 |
|
|
|
|
| 272 |
|
| 273 |
def _mfa_credentials(uname, password):
|
| 274 |
"""The MFA doors share login's credential check and guessing budget, including aliases."""
|
| 275 |
+
# The synthetic runner has no stored factor and is refused without touching tenant #0.
|
| 276 |
+
user = users.qa_identity(uname, password) if (uname and password) else None
|
| 277 |
+
if user:
|
| 278 |
+
raise err(400, "no_stored_account", "That sign in cannot enrol a second factor.")
|
| 279 |
+
throttle_key = _throttle_key(uname)
|
| 280 |
wait = _throttled(throttle_key)
|
| 281 |
if wait:
|
| 282 |
raise err(429, "too_many_attempts",
|
| 283 |
f"too many failed attempts. Try again in {wait} seconds")
|
| 284 |
+
user = users.verify(uname, password) if (uname and password) else None
|
|
|
|
|
|
|
|
|
|
|
|
|
| 285 |
if not user:
|
| 286 |
try:
|
| 287 |
users._hash(password or "x", "00" * 16)
|
|
|
|
| 289 |
pass
|
| 290 |
_note_failure(throttle_key)
|
| 291 |
raise err(401, "invalid_credentials", "that username and password do not match")
|
| 292 |
+
return user, throttle_key
|
|
|
|
|
|
|
| 293 |
|
| 294 |
|
| 295 |
def _mfa_record(uname):
|
|
|
|
| 337 |
if landed is None:
|
| 338 |
# A missing record/no-op write cannot support a cookie or a promise that codes were saved.
|
| 339 |
raise err(503, "store_unavailable", "the second factor was not saved. Try again")
|
| 340 |
+
# Clear only after the whole factor proof and its write succeed, but before minting a cookie.
|
| 341 |
+
_clear_failures(throttle_key)
|
| 342 |
value, _claims = aios_session.mint(tenant, user["username"], int(landed))
|
| 343 |
aios_session.set_cookie(response, request, value)
|
| 344 |
users.touch_login(user["username"])
|
|
|
|
| 347 |
if not users.password_changed_at(rec):
|
| 348 |
# This is a successful first sign-in too: preserve T35's legacy clock backfill.
|
| 349 |
users.stamp_password_age(user["username"])
|
|
|
|
| 350 |
# Responses carrying a new secret or recovery codes must not survive in an HTTP cache.
|
| 351 |
response.headers["Cache-Control"] = "no-store"
|
| 352 |
|
|
|
|
| 360 |
"""
|
| 361 |
body = body or {}
|
| 362 |
uname = str(body.get("username") or "").strip().lower()
|
| 363 |
+
user, _ = _mfa_credentials(uname, str(body.get("password") or ""))
|
| 364 |
rec = _mfa_record(user["username"])
|
| 365 |
if users.mfa_enrolled(rec):
|
| 366 |
raise err(409, "mfa_already_enrolled", _MFA_ALREADY_ENROLLED)
|
|
|
|
| 380 |
"""
|
| 381 |
body = body or {}
|
| 382 |
uname = str(body.get("username") or "").strip().lower()
|
| 383 |
+
user, throttle_key = _mfa_credentials(uname, str(body.get("password") or ""))
|
|
|
|
| 384 |
rec = _mfa_record(user["username"])
|
| 385 |
if users.mfa_enrolled(rec):
|
| 386 |
raise err(409, "mfa_already_enrolled", _MFA_ALREADY_ENROLLED)
|
|
|
|
| 399 |
@router.get("/mfa")
|
| 400 |
def mfa_status(session: Session = Depends(require_session)):
|
| 401 |
"""The calling account's state only: never a secret, code or hash."""
|
| 402 |
+
# This identity has no stored factor; do not read tenant #0 for the QA runner.
|
| 403 |
+
if session.user.get("qa_runner"):
|
| 404 |
rec = {}
|
| 405 |
else:
|
| 406 |
rec = _mfa_record(session.uname)
|
|
|
|
| 418 |
The clear codes appear only in this response; the store and all later reads hold only hashes.
|
| 419 |
"""
|
| 420 |
uname = session.uname
|
| 421 |
+
user, throttle_key = _mfa_credentials(uname, str((body or {}).get("password") or ""))
|
|
|
|
| 422 |
# The session supplies the identity. Body username/tenant fields can never choose a target.
|
| 423 |
rec = _mfa_record(uname)
|
| 424 |
if not users.mfa_enrolled(rec):
|
|
|
|
| 434 |
def password_policy():
|
| 435 |
"""An account-independent checklist the sign-in page can show before authentication."""
|
| 436 |
return {"rules": users.password_rules(), "minLength": users.PASSWORD_MIN_LENGTH,
|
| 437 |
+
"minAgeDays": users.PASSWORD_MIN_AGE_DAYS,
|
| 438 |
"maxAgeDays": users.PASSWORD_MAX_AGE_DAYS, "historyCount": users.PASSWORD_HISTORY}
|
| 439 |
|
| 440 |
|
|
|
|
| 445 |
uname = str(body.get("username") or "").strip().lower()
|
| 446 |
pw = str(body.get("currentPassword") or "")
|
| 447 |
new_pw = str(body.get("newPassword") or "")
|
| 448 |
+
# The staging runner has no stored password and is refused without touching tenant #0.
|
| 449 |
+
user = users.qa_identity(uname, pw) if (uname and pw) else None
|
| 450 |
+
if user:
|
| 451 |
+
raise err(400, "no_stored_password", "This account has no stored password to change here.")
|
| 452 |
+
# Share login's canonical durable bucket: a second unauthenticated door and an email alias
|
| 453 |
+
# must not buy another password-guessing budget.
|
| 454 |
+
throttle_key = _throttle_key(uname)
|
| 455 |
wait = _throttled(throttle_key)
|
| 456 |
if wait:
|
| 457 |
raise err(429, "too_many_attempts",
|
| 458 |
f"too many failed attempts. Try again in {wait} seconds")
|
| 459 |
+
user = users.verify(uname, pw) if (uname and pw) else None
|
|
|
|
|
|
|
|
|
|
|
|
|
| 460 |
if not user:
|
| 461 |
try:
|
| 462 |
users._hash(pw or "x", "00" * 16)
|
|
|
|
| 464 |
pass
|
| 465 |
_note_failure(throttle_key)
|
| 466 |
raise err(401, "invalid_credentials", "that username and password do not match")
|
|
|
|
|
|
|
| 467 |
try:
|
| 468 |
# verify accepts email addresses too. Rotate the RESOLVED registry key, never the typed
|
| 469 |
# alias, or this door could return 204 without changing any password or revoking sessions.
|
api/routes_keychain.py
CHANGED
|
@@ -784,16 +784,18 @@ def _credential_revocation(row, rt):
|
|
| 784 |
def _revocation_residue(entry_type):
|
| 785 |
"""Standing rule 1 requires the cause AND a fix for the mirror this route cannot clear.
|
| 786 |
|
| 787 |
-
|
| 788 |
-
|
|
|
|
| 789 |
"""
|
| 790 |
if entry_type != "amazon_sp":
|
| 791 |
return []
|
| 792 |
return [{
|
| 793 |
"where": "The tenant DuckDB mirror",
|
| 794 |
-
"why": "
|
| 795 |
-
"fix": "
|
| 796 |
-
"platform/harness/amazon_store.py
|
|
|
|
| 797 |
}]
|
| 798 |
|
| 799 |
|
|
|
|
| 784 |
def _revocation_residue(entry_type):
|
| 785 |
"""Standing rule 1 requires the cause AND a fix for the mirror this route cannot clear.
|
| 786 |
|
| 787 |
+
Clearing the app's materialised copy is not evidence that the raw reports disappeared. Mirror
|
| 788 |
+
deletion stays operator-only because it is permanent, exact-tenant work outside the product
|
| 789 |
+
store transaction.
|
| 790 |
"""
|
| 791 |
if entry_type != "amazon_sp":
|
| 792 |
return []
|
| 793 |
return [{
|
| 794 |
"where": "The tenant DuckDB mirror",
|
| 795 |
+
"why": "Connection removal clears product databases but not the raw Amazon mirror.",
|
| 796 |
+
"fix": "After every Amazon credential is revoked, an operator must run "
|
| 797 |
+
"platform/harness/amazon_store.py --purge --tenant <tenant> "
|
| 798 |
+
"--confirm-tenant <tenant> and retain the residue-free receipt.",
|
| 799 |
}]
|
| 800 |
|
| 801 |
|
api/routes_sheet.py
CHANGED
|
@@ -912,7 +912,8 @@ def _reresolved_for_reader(session: Session, book):
|
|
| 912 |
#: here.
|
| 913 |
if wid and scope_key:
|
| 914 |
try:
|
| 915 |
-
sheets.fill_scope_cache(store, wid, cell_key, scope_key, out["value"], stamp
|
|
|
|
| 916 |
except Exception: # noqa: BLE001
|
| 917 |
pass
|
| 918 |
if not touched:
|
|
@@ -2118,7 +2119,7 @@ def compute_cell(workbook_id: str, addr: str, body: dict = Body(default=None),
|
|
| 2118 |
_binding_scope_key(session,
|
| 2119 |
str(binding.get("database") or ""),
|
| 2120 |
binding),
|
| 2121 |
-
resolved["value"], stamp)
|
| 2122 |
except Exception: # noqa: BLE001
|
| 2123 |
pass
|
| 2124 |
|
|
|
|
| 912 |
#: here.
|
| 913 |
if wid and scope_key:
|
| 914 |
try:
|
| 915 |
+
sheets.fill_scope_cache(store, wid, cell_key, scope_key, out["value"], stamp,
|
| 916 |
+
out["database"])
|
| 917 |
except Exception: # noqa: BLE001
|
| 918 |
pass
|
| 919 |
if not touched:
|
|
|
|
| 2119 |
_binding_scope_key(session,
|
| 2120 |
str(binding.get("database") or ""),
|
| 2121 |
binding),
|
| 2122 |
+
resolved["value"], stamp, resolved["database"])
|
| 2123 |
except Exception: # noqa: BLE001
|
| 2124 |
pass
|
| 2125 |
|
platform/core/auth_google.py
CHANGED
|
@@ -11,8 +11,8 @@ DESIGN (recorded now so the full build has a contract to meet):
|
|
| 11 |
core/users.py account via its `email` attribute (set_access(..., email=...)); an
|
| 12 |
unknown email FAILS CLOSED with "no account for this Google identity" β Google login
|
| 13 |
never creates users, so the per-BU/per-module grant model stays the only door.
|
| 14 |
-
- The password gate stays
|
| 15 |
-
|
| 16 |
|
| 17 |
ENV (placeholders β see .env.example additions in docs/GOOGLE_LOGIN_TODO.md):
|
| 18 |
GOOGLE_OAUTH_CLIENT_ID OAuth client id from Google Cloud console
|
|
|
|
| 11 |
core/users.py account via its `email` attribute (set_access(..., email=...)); an
|
| 12 |
unknown email FAILS CLOSED with "no account for this Google identity" β Google login
|
| 13 |
never creates users, so the per-BU/per-module grant model stays the only door.
|
| 14 |
+
- The password gate stays beside it. APP_PASSWORD may seed a fresh registry, but it has no
|
| 15 |
+
authentication bypass; removing stored-password auth is a separate, owner-approved step.
|
| 16 |
|
| 17 |
ENV (placeholders β see .env.example additions in docs/GOOGLE_LOGIN_TODO.md):
|
| 18 |
GOOGLE_OAUTH_CLIENT_ID OAuth client id from Google Cloud console
|
platform/core/sheets.py
CHANGED
|
@@ -69,6 +69,14 @@ WORKBOOK_KEY_PREFIX = "sheet_workbook_"
|
|
| 69 |
#: The companion pointer list. See the module note for why a per-document key needs one.
|
| 70 |
INDEX_KEY = "sheet_workbooks"
|
| 71 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 72 |
# Document format version, independent of the optimistic save revision (`rev`).
|
| 73 |
SHEET_SCHEMA_VERSION = 1
|
| 74 |
DOCUMENT_MAX_COLS = 16384
|
|
@@ -1347,6 +1355,67 @@ def new_workbook(name="", created_by="", workbook_id=""):
|
|
| 1347 |
}
|
| 1348 |
|
| 1349 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1350 |
def create_workbook(st, name="", created_by=""):
|
| 1351 |
"""Write a new `sheet_workbook_<id>` document and register it in the index."""
|
| 1352 |
book = new_workbook(name=name, created_by=created_by)
|
|
@@ -1408,11 +1477,21 @@ def put_workbook(st, workbook_id, raw, expect_rev=None):
|
|
| 1408 |
"That workbook changed somewhere else, so this save was not applied.")
|
| 1409 |
book = migrate_workbook(prior)
|
| 1410 |
# Immutable ownership and server read projections cannot be reassigned by a PUT.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1411 |
protected = {"id", "created_by", "created_at", "updated_at", "rev", "refreshed_at",
|
| 1412 |
-
"sheets", "name", "refreshedFor", "shares", "role", "permissions"
|
|
|
|
| 1413 |
book.update({key: value for key, value in incoming.items() if key not in protected})
|
| 1414 |
book["name"] = name or _text(prior.get("name"), _MAX_NAME) or "Untitled workbook"
|
| 1415 |
book["sheets"] = sheets
|
|
|
|
|
|
|
|
|
|
| 1416 |
book["rev"] = int(prior.get("rev") or 0) + 1
|
| 1417 |
if refreshed_at is not None:
|
| 1418 |
book["refreshed_at"] = _text(refreshed_at, 120) or None
|
|
@@ -1589,6 +1668,7 @@ def patch_cells(st, workbook_id, tab_ref, raw_cells, expect_rev=None):
|
|
| 1589 |
if tab["cols"] > DOCUMENT_MAX_COLS or tab["rows"] > DOCUMENT_MAX_ROWS:
|
| 1590 |
raise SheetRefusal("sheet_extent_out_of_range", "That cell is outside Excel's sheet dimensions.")
|
| 1591 |
book["sheets"] = tabs
|
|
|
|
| 1592 |
book["rev"] = current + 1
|
| 1593 |
book["updated_at"] = _now()
|
| 1594 |
_json_safe(book, "workbook")
|
|
@@ -1677,13 +1757,18 @@ def cached_scope_value(st, workbook_id, cell_key, scope_key):
|
|
| 1677 |
return {"value": slot.get("value"), "refreshedAt": slot.get("refreshedAt")}
|
| 1678 |
|
| 1679 |
|
| 1680 |
-
def fill_scope_cache(st, workbook_id, cell_key, scope_key, value, refreshed_at):
|
| 1681 |
"""Write ONE scope's slot for ONE cell. Best effort: a bad workbook id is a no-op, not a raise.
|
| 1682 |
|
| 1683 |
-
ββ THIS IS THE ONLY DOOR THAT WRITES A SLOT, and
|
| 1684 |
-
reader's
|
| 1685 |
-
|
| 1686 |
-
THIS
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1687 |
"""
|
| 1688 |
try:
|
| 1689 |
wid = _slug_id(workbook_id)
|
|
@@ -1698,6 +1783,7 @@ def fill_scope_cache(st, workbook_id, cell_key, scope_key, value, refreshed_at):
|
|
| 1698 |
slots[scope_key] = {"value": value, "refreshedAt": refreshed_at}
|
| 1699 |
cells[cell_key] = slots
|
| 1700 |
doc["cells"] = cells
|
|
|
|
| 1701 |
return doc, None
|
| 1702 |
|
| 1703 |
_persist(st, SCOPE_CACHE_KEY_PREFIX + wid, _apply)
|
|
|
|
| 69 |
#: The companion pointer list. See the module note for why a per-document key needs one.
|
| 70 |
INDEX_KEY = "sheet_workbooks"
|
| 71 |
|
| 72 |
+
#: DPP 1.8's durable origin vocabulary for Amazon-derived Sheet documents. The analytical
|
| 73 |
+
#: database key is retained because one workbook may read several Amazon reports; a bare
|
| 74 |
+
#: ``amazon_sp`` tag would identify the connector but lose which materialised source supplied the
|
| 75 |
+
#: values. This deliberately lives in the document layer rather than in ``amazon_relational``:
|
| 76 |
+
#: Sheet persistence is shared core code and must not import the API's connector implementation.
|
| 77 |
+
AMAZON_DATABASE_PREFIX = "ut_amzn_"
|
| 78 |
+
AMAZON_DATA_ORIGIN_KEY = "amazon_sp"
|
| 79 |
+
AMAZON_DATA_ORIGIN_NAME = "Amazon Services API"
|
| 80 |
# Document format version, independent of the optimistic save revision (`rev`).
|
| 81 |
SHEET_SCHEMA_VERSION = 1
|
| 82 |
DOCUMENT_MAX_COLS = 16384
|
|
|
|
| 1355 |
}
|
| 1356 |
|
| 1357 |
|
| 1358 |
+
def data_origin_for_database(database):
|
| 1359 |
+
"""Canonical durable origin tag for a known connector database, or ``None``.
|
| 1360 |
+
|
| 1361 |
+
DPP 1.8 permits separation or a mechanism that tags the source of Amazon Information. A
|
| 1362 |
+
workbook and its row-scope cache are shared documents, so they use the tagging arm: every
|
| 1363 |
+
``ut_amzn_*`` source is named explicitly and retained after the binding itself is removed.
|
| 1364 |
+
|
| 1365 |
+
The helper is intentionally conservative. It recognises the reserved Amazon database
|
| 1366 |
+
namespace only; it never guesses that an unrelated database is Amazon from a display label.
|
| 1367 |
+
"""
|
| 1368 |
+
database = str(database or "").strip()
|
| 1369 |
+
if not database.startswith(AMAZON_DATABASE_PREFIX):
|
| 1370 |
+
return None
|
| 1371 |
+
return {
|
| 1372 |
+
"key": AMAZON_DATA_ORIGIN_KEY,
|
| 1373 |
+
"origin": AMAZON_DATA_ORIGIN_NAME,
|
| 1374 |
+
"database": database,
|
| 1375 |
+
}
|
| 1376 |
+
|
| 1377 |
+
|
| 1378 |
+
def _merge_data_origins(doc, *, sheets=(), databases=()):
|
| 1379 |
+
"""Canonicalise and monotonically extend one document's ``dataOrigins`` inventory.
|
| 1380 |
+
|
| 1381 |
+
``dataOrigins`` is SERVER-MAINTAINED: PUT bodies never supply it directly. Only a prior
|
| 1382 |
+
canonical tag, a stored binding/source, or an explicit database from a server-side resolve can
|
| 1383 |
+
add an origin. Existing origins are retained deliberately, even when a user later deletes the
|
| 1384 |
+
binding, because the workbook can still hold pasted values and formulas derived from it.
|
| 1385 |
+
"""
|
| 1386 |
+
found = set()
|
| 1387 |
+
for raw in (doc.get("dataOrigins") or ()) if isinstance(doc, dict) else ():
|
| 1388 |
+
if not isinstance(raw, dict):
|
| 1389 |
+
continue
|
| 1390 |
+
tag = data_origin_for_database(raw.get("database"))
|
| 1391 |
+
if (tag is not None
|
| 1392 |
+
and raw.get("key") == tag["key"]
|
| 1393 |
+
and raw.get("origin") == tag["origin"]):
|
| 1394 |
+
found.add(tag["database"])
|
| 1395 |
+
for database in databases or ():
|
| 1396 |
+
tag = data_origin_for_database(database)
|
| 1397 |
+
if tag is not None:
|
| 1398 |
+
found.add(tag["database"])
|
| 1399 |
+
for tab in sheets or ():
|
| 1400 |
+
if not isinstance(tab, dict):
|
| 1401 |
+
continue
|
| 1402 |
+
for cell in (tab.get("cells") or {}).values():
|
| 1403 |
+
if not isinstance(cell, dict):
|
| 1404 |
+
continue
|
| 1405 |
+
for member in ("binding", "source"):
|
| 1406 |
+
ref = cell.get(member)
|
| 1407 |
+
tag = data_origin_for_database(
|
| 1408 |
+
ref.get("database") if isinstance(ref, dict) else None)
|
| 1409 |
+
if tag is not None:
|
| 1410 |
+
found.add(tag["database"])
|
| 1411 |
+
origins = [data_origin_for_database(database) for database in sorted(found)]
|
| 1412 |
+
if origins:
|
| 1413 |
+
doc["dataOrigins"] = origins
|
| 1414 |
+
else:
|
| 1415 |
+
doc.pop("dataOrigins", None)
|
| 1416 |
+
return origins
|
| 1417 |
+
|
| 1418 |
+
|
| 1419 |
def create_workbook(st, name="", created_by=""):
|
| 1420 |
"""Write a new `sheet_workbook_<id>` document and register it in the index."""
|
| 1421 |
book = new_workbook(name=name, created_by=created_by)
|
|
|
|
| 1477 |
"That workbook changed somewhere else, so this save was not applied.")
|
| 1478 |
book = migrate_workbook(prior)
|
| 1479 |
# Immutable ownership and server read projections cannot be reassigned by a PUT.
|
| 1480 |
+
# ``dataOrigins`` is SERVER-MAINTAINED (DPP 1.8) and therefore protected: a PUT body may
|
| 1481 |
+
# neither supply it nor clear it. It is listed here because this blanket copy of the
|
| 1482 |
+
# incoming document would otherwise hand the client both halves of the control β sending
|
| 1483 |
+
# ``[]`` erases a genuine Amazon origin, and sending a well-formed tag plants one on an
|
| 1484 |
+
# unrelated workbook. The origin list is rebuilt from the PRIOR document plus the cleaned
|
| 1485 |
+
# cells by ``_merge_data_origins`` immediately below.
|
| 1486 |
protected = {"id", "created_by", "created_at", "updated_at", "rev", "refreshed_at",
|
| 1487 |
+
"sheets", "name", "refreshedFor", "shares", "role", "permissions",
|
| 1488 |
+
"dataOrigins"}
|
| 1489 |
book.update({key: value for key, value in incoming.items() if key not in protected})
|
| 1490 |
book["name"] = name or _text(prior.get("name"), _MAX_NAME) or "Untitled workbook"
|
| 1491 |
book["sheets"] = sheets
|
| 1492 |
+
# DPP 1.8: merge from the PRIOR document and the cleaned cells. ``raw.dataOrigins`` is
|
| 1493 |
+
# ignored, so a client cannot erase or replace the server-maintained attribution list.
|
| 1494 |
+
_merge_data_origins(book, sheets=sheets)
|
| 1495 |
book["rev"] = int(prior.get("rev") or 0) + 1
|
| 1496 |
if refreshed_at is not None:
|
| 1497 |
book["refreshed_at"] = _text(refreshed_at, 120) or None
|
|
|
|
| 1668 |
if tab["cols"] > DOCUMENT_MAX_COLS or tab["rows"] > DOCUMENT_MAX_ROWS:
|
| 1669 |
raise SheetRefusal("sheet_extent_out_of_range", "That cell is outside Excel's sheet dimensions.")
|
| 1670 |
book["sheets"] = tabs
|
| 1671 |
+
_merge_data_origins(book, sheets=tabs)
|
| 1672 |
book["rev"] = current + 1
|
| 1673 |
book["updated_at"] = _now()
|
| 1674 |
_json_safe(book, "workbook")
|
|
|
|
| 1757 |
return {"value": slot.get("value"), "refreshedAt": slot.get("refreshedAt")}
|
| 1758 |
|
| 1759 |
|
| 1760 |
+
def fill_scope_cache(st, workbook_id, cell_key, scope_key, value, refreshed_at, database):
|
| 1761 |
"""Write ONE scope's slot for ONE cell. Best effort: a bad workbook id is a no-op, not a raise.
|
| 1762 |
|
| 1763 |
+
ββ THIS IS THE ONLY DOOR THAT WRITES A SLOT, and its callers are server-side resolve paths:
|
| 1764 |
+
a reader's in-memory re-resolve and the one-cell compute door, each immediately after
|
| 1765 |
+
`_resolve_binding` answers under that caller's own scope. `value`, `refreshed_at` and
|
| 1766 |
+
`database` are always what THIS server just computed - never client-supplied, because nothing
|
| 1767 |
+
here parses a request body.
|
| 1768 |
+
|
| 1769 |
+
DPP 1.8: this cache is a SECOND durable document containing the resolved value, not an
|
| 1770 |
+
implementation detail inside the workbook. Its own top-level origin inventory therefore has
|
| 1771 |
+
to name Amazon independently; relying on the opaque scope hash would not identify a source.
|
| 1772 |
"""
|
| 1773 |
try:
|
| 1774 |
wid = _slug_id(workbook_id)
|
|
|
|
| 1783 |
slots[scope_key] = {"value": value, "refreshedAt": refreshed_at}
|
| 1784 |
cells[cell_key] = slots
|
| 1785 |
doc["cells"] = cells
|
| 1786 |
+
_merge_data_origins(doc, databases=(database,))
|
| 1787 |
return doc, None
|
| 1788 |
|
| 1789 |
_persist(st, SCOPE_CACHE_KEY_PREFIX + wid, _apply)
|
platform/core/users.py
CHANGED
|
@@ -1,8 +1,9 @@
|
|
| 1 |
"""Per-user accounts for the platform, persisted in the HF Dataset store (users.json).
|
| 2 |
|
| 3 |
Passwords are salted + PBKDF2-HMAC-SHA256 (200k iterations) β never stored or logged in plaintext.
|
| 4 |
-
A bootstrap 'admin' account is seeded from APP_PASSWORD
|
| 5 |
-
|
|
|
|
| 6 |
|
| 7 |
Each account carries BU access ('all' or a list of team-ids [5=Fisch, 6=Royal]) which drives
|
| 8 |
allowed_bus() β the basis for per-Business-Unit permissioning (a Royal-only user never sees Fisch).
|
|
@@ -10,6 +11,7 @@ allowed_bus() β the basis for per-Business-Unit permissioning (a Royal-only us
|
|
| 10 |
import os
|
| 11 |
import hmac
|
| 12 |
import hashlib
|
|
|
|
| 13 |
import secrets
|
| 14 |
import re
|
| 15 |
import base64
|
|
@@ -78,12 +80,23 @@ def _hash(pw, salt):
|
|
| 78 |
|
| 79 |
# ------------------------------------------------------------------ password policy (W57-T35 / C12)
|
| 80 |
# ONE policy for account creation and both rotation doors. Login only reads the age: checking
|
| 81 |
-
# history there would spend
|
| 82 |
PASSWORD_MIN_LENGTH = 12
|
| 83 |
PASSWORD_MAX_LENGTH = 256 # bound PBKDF2 input, not a claim about strength
|
| 84 |
-
PASSWORD_HISTORY =
|
|
|
|
| 85 |
PASSWORD_MAX_AGE_DAYS = 90
|
| 86 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 87 |
|
| 88 |
class PasswordPolicyError(ValueError):
|
| 89 |
"""Raised with FAILED RULES attached, never a bare message.
|
|
@@ -109,6 +122,8 @@ def password_rules():
|
|
| 109 |
{'rule': 'identifier',
|
| 110 |
'message': 'it must not contain the username, the email local part, or any word of '
|
| 111 |
'the display name that is 3 characters or longer'},
|
|
|
|
|
|
|
| 112 |
{'rule': 'reuse',
|
| 113 |
'message': f'it must not repeat the current password or any of the last '
|
| 114 |
f'{PASSWORD_HISTORY} passwords'},
|
|
@@ -116,7 +131,8 @@ def password_rules():
|
|
| 116 |
]
|
| 117 |
|
| 118 |
|
| 119 |
-
def password_failures(pw, username=None, name=None, email=None, record=None
|
|
|
|
| 120 |
"""Every rule this password breaks, in policy order. Empty list means it passes."""
|
| 121 |
pw = str(pw or '')
|
| 122 |
# Case-fold identifiers, but never the credential being hashed. Name punctuation separates
|
|
@@ -135,6 +151,20 @@ def password_failures(pw, username=None, name=None, email=None, record=None):
|
|
| 135 |
'symbol': not any(not c.isalnum() for c in pw),
|
| 136 |
'identifier': any(identifier and identifier in folded for identifier in identifiers),
|
| 137 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 138 |
# An oversized value is already refused; do not feed it to PBKDF2 merely to describe reuse.
|
| 139 |
# Every history entry owns its own salt, so comparing against only today's salt proves nothing.
|
| 140 |
if record is not None and len(pw) <= PASSWORD_MAX_LENGTH:
|
|
@@ -161,13 +191,16 @@ def password_changed_at(record):
|
|
| 161 |
return stamp
|
| 162 |
|
| 163 |
|
| 164 |
-
def password_age_days(record):
|
| 165 |
"""Fractional days since the last password change, or None without a readable stamp."""
|
| 166 |
import datetime as _dt
|
| 167 |
stamp = password_changed_at(record)
|
| 168 |
if stamp is None:
|
| 169 |
return None
|
| 170 |
-
|
|
|
|
|
|
|
|
|
|
| 171 |
|
| 172 |
|
| 173 |
def password_expired(record):
|
|
@@ -401,14 +434,132 @@ def _record(pw, name, role, bus, active=True, modules='all', agent=None, email=N
|
|
| 401 |
return rec
|
| 402 |
|
| 403 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 404 |
def registry():
|
| 405 |
return store.get('users')
|
| 406 |
|
| 407 |
|
| 408 |
def ensure_bootstrap():
|
| 409 |
"""Seed an 'admin' account from APP_PASSWORD ONLY on a truly fresh store (no users file yet).
|
| 410 |
-
|
| 411 |
-
|
|
|
|
|
|
|
| 412 |
|
| 413 |
Critically, this NEVER overwrites an existing registry: it seeds only when store.exists('users')
|
| 414 |
is definitively False. A transient read failure at startup used to return {} and make this
|
|
@@ -426,8 +577,8 @@ def ensure_bootstrap():
|
|
| 426 |
master = os.environ.get('APP_PASSWORD', '')
|
| 427 |
if not master:
|
| 428 |
return
|
| 429 |
-
# Keep the fresh-store
|
| 430 |
-
# APP_PASSWORD holds here could leave a new deployment with nobody able to
|
| 431 |
try:
|
| 432 |
store.put('users', {'admin': _record(master, 'Administrator', 'admin', 'all')})
|
| 433 |
except Exception:
|
|
@@ -480,8 +631,8 @@ def epoch(username):
|
|
| 480 |
|
| 481 |
None is NOT 0. 0 is "this account exists and has never been revoked"; None is "no record" β
|
| 482 |
which the session verifier must treat as a reason to refuse, not as a default to compare
|
| 483 |
-
against.
|
| 484 |
-
|
| 485 |
"""
|
| 486 |
username = (username or '').strip().lower()
|
| 487 |
try:
|
|
@@ -533,18 +684,20 @@ def bump_epoch(username):
|
|
| 533 |
|
| 534 |
|
| 535 |
def verify(username, pw):
|
| 536 |
-
"""Return a public user
|
| 537 |
-
|
|
|
|
|
|
|
|
|
|
| 538 |
username = (username or '').strip().lower()
|
| 539 |
if not username or not pw:
|
| 540 |
return None
|
| 541 |
-
master = os.environ.get('APP_PASSWORD', '')
|
| 542 |
try:
|
| 543 |
# read fresh so accounts created moments ago (UI or out-of-band) are recognised at once
|
| 544 |
reg = store.get('users', fresh=True)
|
| 545 |
except Exception:
|
| 546 |
reg = {}
|
| 547 |
-
u =
|
| 548 |
# β KEY FIRST, EMAIL SECOND, AND SINCE WAVE 37 THAT ORDER IS A DECISION RATHER THAN AN ACCIDENT.
|
| 549 |
# R8 lets a username BE an email address, so a typed string can now match a registry KEY and a
|
| 550 |
# different account's `email` field at the same time. The key wins, here, by construction: this
|
|
@@ -552,40 +705,8 @@ def verify(username, pw):
|
|
| 552 |
# to CREATE either collision (`username_shadows_email` / `email_shadows_username`) so the
|
| 553 |
# ambiguity cannot be introduced through the product; this line is what decides it for any
|
| 554 |
# record that arrived some other way.
|
| 555 |
-
if u is None and '@' in username:
|
| 556 |
-
# Wave 18 (R1): the login box takes a username OR an email β admin@nurilab.id signs in
|
| 557 |
-
# without knowing the slug an admin chose. First case-insensitive email match wins;
|
| 558 |
-
# ambiguity is an admin data problem, not a login feature.
|
| 559 |
-
for k, r in reg.items():
|
| 560 |
-
if isinstance(r, dict) and str(r.get('email') or '').strip().lower() == username:
|
| 561 |
-
username, u = k, r
|
| 562 |
-
break
|
| 563 |
if u and u.get('active', True) and hmac.compare_digest(_hash(pw, u['salt']), u['hash']):
|
| 564 |
return _public(username, u)
|
| 565 |
-
# emergency master: admin + APP_PASSWORD always works (covers first run / store outage)
|
| 566 |
-
if username == 'admin' and master and hmac.compare_digest(str(pw), master):
|
| 567 |
-
# β CARRY THE RECORD'S CURRENT EPOCH when there is a record to read, so the session cookie
|
| 568 |
-
# the API mints from this dict AGREES with the stored account.
|
| 569 |
-
#
|
| 570 |
-
# This is not what stops the emergency lockout β `deps._user_for`'s master fallback does
|
| 571 |
-
# that, and a negative control confirmed the lockout is gone with or without this line.
|
| 572 |
-
# What it fixes is subtler and is a SCOPE question: a cookie whose epoch disagrees with the
|
| 573 |
-
# record falls through to that master fallback, which hands back a SYNTHETIC identity
|
| 574 |
-
# (`bus: 'all'`, `modules: 'all'`). An admin whose record narrows either field would
|
| 575 |
-
# therefore be silently WIDENED to consolidated, all-module access for the life of that
|
| 576 |
-
# session. Matching the epoch means the record branch wins and the account's real scope
|
| 577 |
-
# applies, leaving the master fallback as the true last resort it is meant to be.
|
| 578 |
-
# 0 when the store is unreachable, which is the case this branch was written for.
|
| 579 |
-
#
|
| 580 |
-
# β WAVE 40 (R2) β `emergency_master` MARKS THIS DICT SO THE LOGIN CAN SKIP THE EPOCH BUMP.
|
| 581 |
-
# Every ordinary sign-in now bumps the account's epoch to sign its other sessions out, but
|
| 582 |
-
# this identity is deliberately NOT epoch-revocable: `deps._user_for`'s master fallback
|
| 583 |
-
# admits it whatever the record says. A bump here would therefore revoke nothing, while
|
| 584 |
-
# forcing a store WRITE on the one code path that exists for a store outage. Internal to
|
| 585 |
-
# the server: `routes_auth._public_user` is a whitelist projection and does not name it.
|
| 586 |
-
return {'username': 'admin', 'name': 'Administrator', 'role': 'admin', 'bus': 'all',
|
| 587 |
-
'modules': 'all', 'epoch': int((reg.get('admin') or {}).get('epoch') or 0),
|
| 588 |
-
'emergency_master': True}
|
| 589 |
return None
|
| 590 |
|
| 591 |
|
|
|
|
| 1 |
"""Per-user accounts for the platform, persisted in the HF Dataset store (users.json).
|
| 2 |
|
| 3 |
Passwords are salted + PBKDF2-HMAC-SHA256 (200k iterations) β never stored or logged in plaintext.
|
| 4 |
+
A bootstrap 'admin' account is seeded from APP_PASSWORD only when the registry is genuinely empty.
|
| 5 |
+
After that seed, APP_PASSWORD has no bypass semantics: every production sign-in resolves a stored,
|
| 6 |
+
active account and is subject to password age, durable lockout, and MFA at the API boundary.
|
| 7 |
|
| 8 |
Each account carries BU access ('all' or a list of team-ids [5=Fisch, 6=Royal]) which drives
|
| 9 |
allowed_bus() β the basis for per-Business-Unit permissioning (a Royal-only user never sees Fisch).
|
|
|
|
| 11 |
import os
|
| 12 |
import hmac
|
| 13 |
import hashlib
|
| 14 |
+
import math
|
| 15 |
import secrets
|
| 16 |
import re
|
| 17 |
import base64
|
|
|
|
| 80 |
|
| 81 |
# ------------------------------------------------------------------ password policy (W57-T35 / C12)
|
| 82 |
# ONE policy for account creation and both rotation doors. Login only reads the age: checking
|
| 83 |
+
# history there would spend eleven PBKDF2 passes on every sign-in without improving authentication.
|
| 84 |
PASSWORD_MIN_LENGTH = 12
|
| 85 |
PASSWORD_MAX_LENGTH = 256 # bound PBKDF2 input, not a claim about strength
|
| 86 |
+
PASSWORD_HISTORY = 10
|
| 87 |
+
PASSWORD_MIN_AGE_DAYS = 1
|
| 88 |
PASSWORD_MAX_AGE_DAYS = 90
|
| 89 |
|
| 90 |
+
# Amazon SP-API's current account-control baseline: no more than ten failed attempts, followed by
|
| 91 |
+
# a thirty-minute suspension. This state is a STORE document rather than a module dict so several
|
| 92 |
+
# workers and a deploy see one budget. The live PostgreSQL backend serialises update() with
|
| 93 |
+
# SELECT ... FOR UPDATE; the HF compatibility backend retains its existing sync-update contract.
|
| 94 |
+
LOGIN_LOCKOUT_AFTER = 10
|
| 95 |
+
LOGIN_LOCKOUT_SECONDS = 30 * 60
|
| 96 |
+
LOGIN_LOCKOUT_BUCKET = 'auth_login_failures'
|
| 97 |
+
_LOGIN_LOCKOUT_MAX_TRACKED = 2048
|
| 98 |
+
_LOGIN_LOCKOUT_RETENTION_SECONDS = 7 * 24 * 60 * 60
|
| 99 |
+
|
| 100 |
|
| 101 |
class PasswordPolicyError(ValueError):
|
| 102 |
"""Raised with FAILED RULES attached, never a bare message.
|
|
|
|
| 122 |
{'rule': 'identifier',
|
| 123 |
'message': 'it must not contain the username, the email local part, or any word of '
|
| 124 |
'the display name that is 3 characters or longer'},
|
| 125 |
+
{'rule': 'minimum_age',
|
| 126 |
+
'message': f'it can normally be changed only after {PASSWORD_MIN_AGE_DAYS} day'},
|
| 127 |
{'rule': 'reuse',
|
| 128 |
'message': f'it must not repeat the current password or any of the last '
|
| 129 |
f'{PASSWORD_HISTORY} passwords'},
|
|
|
|
| 131 |
]
|
| 132 |
|
| 133 |
|
| 134 |
+
def password_failures(pw, username=None, name=None, email=None, record=None,
|
| 135 |
+
allow_early_rotation=False, now=None):
|
| 136 |
"""Every rule this password breaks, in policy order. Empty list means it passes."""
|
| 137 |
pw = str(pw or '')
|
| 138 |
# Case-fold identifiers, but never the credential being hashed. Name punctuation separates
|
|
|
|
| 151 |
'symbol': not any(not c.isalnum() for c in pw),
|
| 152 |
'identifier': any(identifier and identifier in folded for identifier in identifiers),
|
| 153 |
}
|
| 154 |
+
# Amazon DPP 1.4.1's one-day minimum age prevents rapid rotations from walking a password out
|
| 155 |
+
# of the ten-entry history. The first activation/legacy rotation has no history and stays
|
| 156 |
+
# available; an administrator-forced reset marks must_change_password and likewise remains an
|
| 157 |
+
# emergency recovery path. The separate admin password route passes allow_early_rotation only
|
| 158 |
+
# after its role and tenant walls, for a direct suspected-compromise reset.
|
| 159 |
+
age = password_age_days(record, now=now) if record is not None else None
|
| 160 |
+
broken['minimum_age'] = (
|
| 161 |
+
record is not None
|
| 162 |
+
and not allow_early_rotation
|
| 163 |
+
and (record or {}).get('must_change_password') is not True
|
| 164 |
+
and bool((record or {}).get('password_history'))
|
| 165 |
+
and age is not None
|
| 166 |
+
and age < PASSWORD_MIN_AGE_DAYS
|
| 167 |
+
)
|
| 168 |
# An oversized value is already refused; do not feed it to PBKDF2 merely to describe reuse.
|
| 169 |
# Every history entry owns its own salt, so comparing against only today's salt proves nothing.
|
| 170 |
if record is not None and len(pw) <= PASSWORD_MAX_LENGTH:
|
|
|
|
| 191 |
return stamp
|
| 192 |
|
| 193 |
|
| 194 |
+
def password_age_days(record, now=None):
|
| 195 |
"""Fractional days since the last password change, or None without a readable stamp."""
|
| 196 |
import datetime as _dt
|
| 197 |
stamp = password_changed_at(record)
|
| 198 |
if stamp is None:
|
| 199 |
return None
|
| 200 |
+
current = now or _dt.datetime.now(_dt.timezone.utc)
|
| 201 |
+
if current.utcoffset() is None:
|
| 202 |
+
current = current.replace(tzinfo=_dt.timezone.utc)
|
| 203 |
+
return (current - _dt.datetime.fromisoformat(stamp)).total_seconds() / 86400
|
| 204 |
|
| 205 |
|
| 206 |
def password_expired(record):
|
|
|
|
| 434 |
return rec
|
| 435 |
|
| 436 |
|
| 437 |
+
def _resolve_identifier(reg, identifier):
|
| 438 |
+
"""Return ``(canonical_username, record_or_none)`` using login's key-first rule."""
|
| 439 |
+
identifier = (identifier or '').strip().lower()
|
| 440 |
+
record = (reg or {}).get(identifier)
|
| 441 |
+
if record is None and '@' in identifier:
|
| 442 |
+
for username, candidate in (reg or {}).items():
|
| 443 |
+
if (isinstance(candidate, dict)
|
| 444 |
+
and str(candidate.get('email') or '').strip().lower() == identifier):
|
| 445 |
+
return str(username).strip().lower(), candidate
|
| 446 |
+
return identifier, record
|
| 447 |
+
|
| 448 |
+
|
| 449 |
+
def canonical_login_identifier(identifier):
|
| 450 |
+
"""Resolve a username/email alias to the one durable lockout bucket for that account.
|
| 451 |
+
|
| 452 |
+
The value is used only as input to a one-way digest before persistence. Unknown identifiers
|
| 453 |
+
retain their normalised spelling, so their responses remain indistinguishable from a bad
|
| 454 |
+
password for a real account.
|
| 455 |
+
"""
|
| 456 |
+
reg = store.get('users', fresh=True) or {}
|
| 457 |
+
canonical, _record_or_none = _resolve_identifier(reg, identifier)
|
| 458 |
+
return canonical
|
| 459 |
+
|
| 460 |
+
|
| 461 |
+
def _login_lockout_key(identifier):
|
| 462 |
+
normalised = (identifier or '').strip().lower()
|
| 463 |
+
return hashlib.sha256(('aios-login-v1\0' + normalised).encode('utf-8')).hexdigest()
|
| 464 |
+
|
| 465 |
+
|
| 466 |
+
def login_lockout_remaining(identifier, now=None):
|
| 467 |
+
"""Seconds left in the shared account lock, or zero.
|
| 468 |
+
|
| 469 |
+
Store failures deliberately propagate. A login route that cannot read the shared budget must
|
| 470 |
+
fail closed rather than silently grant each worker a fresh set of guesses.
|
| 471 |
+
"""
|
| 472 |
+
now = time.time() if now is None else float(now)
|
| 473 |
+
doc = store.get(LOGIN_LOCKOUT_BUCKET, fresh=True) or {}
|
| 474 |
+
hit = doc.get(_login_lockout_key(identifier)) if isinstance(doc, dict) else None
|
| 475 |
+
if not isinstance(hit, dict) or int(hit.get('count') or 0) < LOGIN_LOCKOUT_AFTER:
|
| 476 |
+
return 0
|
| 477 |
+
try:
|
| 478 |
+
remaining = float(hit.get('locked_until') or 0) - now
|
| 479 |
+
except (TypeError, ValueError, OverflowError):
|
| 480 |
+
return 0
|
| 481 |
+
return max(0, int(math.ceil(remaining)))
|
| 482 |
+
|
| 483 |
+
|
| 484 |
+
def note_login_failure(identifier, now=None):
|
| 485 |
+
"""Atomically count one failed password/MFA proof and return lock seconds remaining."""
|
| 486 |
+
now = time.time() if now is None else float(now)
|
| 487 |
+
key = _login_lockout_key(identifier)
|
| 488 |
+
|
| 489 |
+
def _note(doc):
|
| 490 |
+
doc = dict(doc) if isinstance(doc, dict) else {}
|
| 491 |
+
# Keep the security document bounded under random-identifier abuse. An active lock is
|
| 492 |
+
# never pruned, even if a malformed/old ``last_failed_at`` accompanies it.
|
| 493 |
+
retained = {}
|
| 494 |
+
for stored_key, value in doc.items():
|
| 495 |
+
if not isinstance(value, dict):
|
| 496 |
+
continue
|
| 497 |
+
try:
|
| 498 |
+
last = float(value.get('last_failed_at') or 0)
|
| 499 |
+
until = float(value.get('locked_until') or 0)
|
| 500 |
+
except (TypeError, ValueError, OverflowError):
|
| 501 |
+
continue
|
| 502 |
+
if until > now or last >= now - _LOGIN_LOCKOUT_RETENTION_SECONDS:
|
| 503 |
+
retained[stored_key] = value
|
| 504 |
+
doc = retained
|
| 505 |
+
|
| 506 |
+
prior = doc.get(key) or {}
|
| 507 |
+
try:
|
| 508 |
+
count = int(prior.get('count') or 0)
|
| 509 |
+
last = float(prior.get('last_failed_at') or 0)
|
| 510 |
+
until = float(prior.get('locked_until') or 0)
|
| 511 |
+
except (TypeError, ValueError, OverflowError):
|
| 512 |
+
count, last, until = 0, 0.0, 0.0
|
| 513 |
+
# Once a lock expires, or an incomplete failure window ages out, the next miss starts a
|
| 514 |
+
# new budget. Attempts rejected while locked never reach this writer and cannot extend it.
|
| 515 |
+
if until <= now and (count >= LOGIN_LOCKOUT_AFTER or now - last >= LOGIN_LOCKOUT_SECONDS):
|
| 516 |
+
count, until = 0, 0.0
|
| 517 |
+
count += 1
|
| 518 |
+
if count >= LOGIN_LOCKOUT_AFTER:
|
| 519 |
+
until = now + LOGIN_LOCKOUT_SECONDS
|
| 520 |
+
doc[key] = {'count': count, 'last_failed_at': now, 'locked_until': until}
|
| 521 |
+
|
| 522 |
+
if len(doc) > _LOGIN_LOCKOUT_MAX_TRACKED:
|
| 523 |
+
# Discard expired/incomplete budgets before active locks. A pathological set of more
|
| 524 |
+
# than 2,048 simultaneous active identifiers still evicts oldest-first: the document
|
| 525 |
+
# stays bounded rather than becoming an attacker-controlled memory allocation.
|
| 526 |
+
oldest = sorted(doc, key=lambda stored_key: (
|
| 527 |
+
float((doc.get(stored_key) or {}).get('locked_until') or 0) > now,
|
| 528 |
+
float((doc.get(stored_key) or {}).get('last_failed_at') or 0)))
|
| 529 |
+
excess = len(doc) - _LOGIN_LOCKOUT_MAX_TRACKED
|
| 530 |
+
for stored_key in [candidate for candidate in oldest if candidate != key][:excess]:
|
| 531 |
+
doc.pop(stored_key, None)
|
| 532 |
+
return doc
|
| 533 |
+
|
| 534 |
+
landed = store.update(LOGIN_LOCKOUT_BUCKET, _note)
|
| 535 |
+
hit = (landed or {}).get(key) if isinstance(landed, dict) else None
|
| 536 |
+
if not isinstance(hit, dict) or int(hit.get('count') or 0) < LOGIN_LOCKOUT_AFTER:
|
| 537 |
+
return 0
|
| 538 |
+
return max(0, int(math.ceil(float(hit.get('locked_until') or 0) - now)))
|
| 539 |
+
|
| 540 |
+
|
| 541 |
+
def clear_login_failures(identifier):
|
| 542 |
+
"""Clear the account's shared guessing budget after a complete successful proof."""
|
| 543 |
+
key = _login_lockout_key(identifier)
|
| 544 |
+
|
| 545 |
+
def _clear(doc):
|
| 546 |
+
doc = dict(doc) if isinstance(doc, dict) else {}
|
| 547 |
+
doc.pop(key, None)
|
| 548 |
+
return doc
|
| 549 |
+
|
| 550 |
+
return store.update(LOGIN_LOCKOUT_BUCKET, _clear)
|
| 551 |
+
|
| 552 |
+
|
| 553 |
def registry():
|
| 554 |
return store.get('users')
|
| 555 |
|
| 556 |
|
| 557 |
def ensure_bootstrap():
|
| 558 |
"""Seed an 'admin' account from APP_PASSWORD ONLY on a truly fresh store (no users file yet).
|
| 559 |
+
|
| 560 |
+
Idempotent and a no-op if the store is unavailable. There is intentionally no authentication
|
| 561 |
+
fallback: after the seed, the stored account is the sole production identity and must enrol
|
| 562 |
+
MFA like every other account.
|
| 563 |
|
| 564 |
Critically, this NEVER overwrites an existing registry: it seeds only when store.exists('users')
|
| 565 |
is definitively False. A transient read failure at startup used to return {} and make this
|
|
|
|
| 577 |
master = os.environ.get('APP_PASSWORD', '')
|
| 578 |
if not master:
|
| 579 |
return
|
| 580 |
+
# Keep the fresh-store bootstrap seed exempt from the password policy. Rejecting whatever
|
| 581 |
+
# APP_PASSWORD holds here could leave a new deployment with nobody able to begin enrolment.
|
| 582 |
try:
|
| 583 |
store.put('users', {'admin': _record(master, 'Administrator', 'admin', 'all')})
|
| 584 |
except Exception:
|
|
|
|
| 631 |
|
| 632 |
None is NOT 0. 0 is "this account exists and has never been revoked"; None is "no record" β
|
| 633 |
which the session verifier must treat as a reason to refuse, not as a default to compare
|
| 634 |
+
against. A username with no stored account is always refused; a signed cookie is not an
|
| 635 |
+
identity record.
|
| 636 |
"""
|
| 637 |
username = (username or '').strip().lower()
|
| 638 |
try:
|
|
|
|
| 684 |
|
| 685 |
|
| 686 |
def verify(username, pw):
|
| 687 |
+
"""Return a public stored user on success, else ``None``.
|
| 688 |
+
|
| 689 |
+
``APP_PASSWORD`` may have seeded the first admin record, but it is never consulted here. A
|
| 690 |
+
missing/unreachable registry therefore fails closed instead of minting an MFA-exempt identity.
|
| 691 |
+
"""
|
| 692 |
username = (username or '').strip().lower()
|
| 693 |
if not username or not pw:
|
| 694 |
return None
|
|
|
|
| 695 |
try:
|
| 696 |
# read fresh so accounts created moments ago (UI or out-of-band) are recognised at once
|
| 697 |
reg = store.get('users', fresh=True)
|
| 698 |
except Exception:
|
| 699 |
reg = {}
|
| 700 |
+
username, u = _resolve_identifier(reg, username)
|
| 701 |
# β KEY FIRST, EMAIL SECOND, AND SINCE WAVE 37 THAT ORDER IS A DECISION RATHER THAN AN ACCIDENT.
|
| 702 |
# R8 lets a username BE an email address, so a typed string can now match a registry KEY and a
|
| 703 |
# different account's `email` field at the same time. The key wins, here, by construction: this
|
|
|
|
| 705 |
# to CREATE either collision (`username_shadows_email` / `email_shadows_username`) so the
|
| 706 |
# ambiguity cannot be introduced through the product; this line is what decides it for any
|
| 707 |
# record that arrived some other way.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 708 |
if u and u.get('active', True) and hmac.compare_digest(_hash(pw, u['salt']), u['hash']):
|
| 709 |
return _public(username, u)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 710 |
return None
|
| 711 |
|
| 712 |
|
platform/harness/amazon_client.py
CHANGED
|
@@ -21,19 +21,17 @@ disregards AWS SigV4 entirely and authorizes on the LWA access token alone: a pl
|
|
| 21 |
signing requests is describing a dead API.
|
| 22 |
Β· https://developer-docs.amazon.com/sp-api/changelog/sp-api-will-no-longer-require-aws-iam-or-aws-signature-version-4
|
| 23 |
|
| 24 |
-
β **READ-ONLY, ENFORCED HERE, WITH EXACTLY
|
| 25 |
GET before it builds a request. SP-API can create feeds, send buyer messages and adjust listings;
|
| 26 |
none of that is reachable from this module, and the refusal sits below every route and payload so no
|
| 27 |
caller can opt out.
|
| 28 |
|
| 29 |
-
β **The
|
| 30 |
-
BUILD a report is a POST, and
|
| 31 |
-
|
| 32 |
-
|
| 33 |
-
|
| 34 |
-
|
| 35 |
-
a price, an order or an inventory level, and it cannot send a buyer anything. Full reasoning sits on
|
| 36 |
-
`_REPORTS_CREATE_PATH`.
|
| 37 |
|
| 38 |
β **THE BULK PATH IS THE REPORTS API, NEVER PAGE-BY-PAGE.** SP-API is token-bucket rate limited per
|
| 39 |
operation per selling-partner/application pair, and Amazon treats throttling as evidence of a wrong
|
|
@@ -75,6 +73,18 @@ REPORT_BURST = 15
|
|
| 75 |
#: https://developer-docs.amazon.com/sp-api/reference/createreport
|
| 76 |
REPORT_SUSTAINED_SECONDS = 60.0
|
| 77 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 78 |
# β One process-wide bucket and lock prevent concurrent syncs from spending the same token.
|
| 79 |
# The contextual wait belongs to the current report_document logger, never another sync's log.
|
| 80 |
_report_pacer = {
|
|
@@ -85,6 +95,13 @@ _report_pacer = {
|
|
| 85 |
"waited": contextvars.ContextVar("amazon_report_create_waited", default=0.0),
|
| 86 |
}
|
| 87 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 88 |
#: β MUST MATCH `core.keychain.ENTRY_TYPES` OR `add_entry` 400s with a message about our data
|
| 89 |
#: structure on a screen about their Amazon account. That was the wave-31 Meta defect and the
|
| 90 |
#: comments in `keychain.py` warn about it in as many words.
|
|
@@ -120,6 +137,14 @@ SANDBOX_ENDPOINTS = {
|
|
| 120 |
}
|
| 121 |
DEFAULT_REGION = "na"
|
| 122 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 123 |
|
| 124 |
def is_sandbox(fields):
|
| 125 |
"""Does this stored credential belong to a SANDBOX app? -> bool.
|
|
@@ -227,7 +252,8 @@ REPORT_TYPES = {
|
|
| 227 |
},
|
| 228 |
"inventory": {
|
| 229 |
"report": "GET_LEDGER_SUMMARY_VIEW_DATA",
|
| 230 |
-
"
|
|
|
|
| 231 |
"role": "Amazon Fulfillment / Amazon Warehousing and Distribution",
|
| 232 |
},
|
| 233 |
# ββ O5 IS CLOSED AND THE ANSWER IS NO (researched 2026-09-04). All 101 report types were
|
|
@@ -246,8 +272,8 @@ REPORT_TYPES = {
|
|
| 246 |
"pricing": {
|
| 247 |
"report": "GET_SALES_AND_TRAFFIC_REPORT",
|
| 248 |
"report_covers": "buy box SHARE only; PRICE has no bulk path",
|
| 249 |
-
"price_path": "
|
| 250 |
-
"validates_against": "sales and traffic report totals
|
| 251 |
"role": "Pricing + Brand Analytics",
|
| 252 |
},
|
| 253 |
"brand_analytics": {
|
|
@@ -305,7 +331,7 @@ def _safe(url):
|
|
| 305 |
return "<url>"
|
| 306 |
|
| 307 |
|
| 308 |
-
def _retry(attempt_fn, what, *, sleep=time.sleep, rand=random.random):
|
| 309 |
"""Retry only SP-API throttling; return successes and propagate other refusals unchanged."""
|
| 310 |
for n in range(1, RETRY_MAX_ATTEMPTS + 1):
|
| 311 |
try:
|
|
@@ -315,10 +341,11 @@ def _retry(attempt_fn, what, *, sleep=time.sleep, rand=random.random):
|
|
| 315 |
# β probe_credential exposes str(e)[:200]; amazon_store.sync stores str(e)[:160].
|
| 316 |
# Throttle, cause AND the complete fix must fit in the first 160 characters;
|
| 317 |
# this head is 152 with the named cap. Operation/URL detail must stay AFTER it.
|
|
|
|
| 318 |
raise AmazonThrottled(
|
| 319 |
f"Amazon throttled (429); {n - 1} retries spent. "
|
| 320 |
"Cause: SP-API limits per operation, app and seller. "
|
| 321 |
-
"Fix:
|
| 322 |
f"Operation: {_safe(what)}", retry_after=e.retry_after) from None
|
| 323 |
wait = min(RETRY_MAX_SECONDS, RETRY_BASE_SECONDS * 2 ** (n - 1))
|
| 324 |
floor = min(RETRY_MAX_SECONDS, e.retry_after or 0.0)
|
|
@@ -348,6 +375,19 @@ def _pace_report_create(*, sleep=time.sleep, now=time.monotonic):
|
|
| 348 |
return wait
|
| 349 |
|
| 350 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 351 |
def _post_form(url, data, timeout=_TIMEOUT):
|
| 352 |
body = urllib.parse.urlencode(data).encode("ascii")
|
| 353 |
req = urllib.request.Request(url, data=body, method="POST")
|
|
@@ -393,7 +433,7 @@ def access_token(fields, timeout=_TIMEOUT):
|
|
| 393 |
|
| 394 |
|
| 395 |
def _request(path, token, region=DEFAULT_REGION, method="GET", timeout=_TIMEOUT,
|
| 396 |
-
sandbox=False, **params):
|
| 397 |
"""One read-only SP-API call. Returns the decoded JSON body.
|
| 398 |
|
| 399 |
β NON-GET IS REFUSED HERE, BEFORE THE REQUEST IS BUILT. There is no flag, no keyword and no
|
|
@@ -435,9 +475,10 @@ def _request(path, token, region=DEFAULT_REGION, method="GET", timeout=_TIMEOUT,
|
|
| 435 |
detail = ""
|
| 436 |
# 429 is the rate limiter, and standing rule 1 wants it NAMED rather than swallowed.
|
| 437 |
if e.code == 429:
|
|
|
|
| 438 |
raise AmazonThrottled(
|
| 439 |
f"{_safe(url)} was throttled (429). SP-API is token-bucket limited per operation; "
|
| 440 |
-
"the fix is
|
| 441 |
retry_after=(e.headers or {}).get("Retry-After")) from None
|
| 442 |
raise AmazonError(f"{_safe(url)} answered {e.code}"
|
| 443 |
+ (f": {detail}" if detail else "")) from None
|
|
@@ -446,10 +487,161 @@ def _request(path, token, region=DEFAULT_REGION, method="GET", timeout=_TIMEOUT,
|
|
| 446 |
|
| 447 |
base = (SANDBOX_ENDPOINTS if sandbox else ENDPOINTS).get(
|
| 448 |
str(region or DEFAULT_REGION).lower(), "")
|
| 449 |
-
return _retry(
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 450 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 451 |
|
| 452 |
-
|
|
|
|
| 453 |
#:
|
| 454 |
#: The Reports API bulk path cannot be reached with GET alone: asking Amazon to BUILD a report is
|
| 455 |
#: `POST /reports/2021-06-30/reports`. Standing rule 1 forbids a silent cap on a connector's
|
|
@@ -463,8 +655,9 @@ def _request(path, token, region=DEFAULT_REGION, method="GET", timeout=_TIMEOUT,
|
|
| 463 |
#: caller-supplied value to smuggle another operation through.
|
| 464 |
#:
|
| 465 |
#: β AND THE BLAST RADIUS IS WORTH STATING PLAINLY, because "read-only" is a promise to the owner
|
| 466 |
-
#: and this is
|
| 467 |
-
#:
|
|
|
|
| 468 |
#: The two roles that could do those things were REFUSED at registration (R7).
|
| 469 |
_REPORTS_CREATE_PATH = "reports/2021-06-30/reports"
|
| 470 |
|
|
@@ -529,7 +722,9 @@ def _create_report(token, report_type, marketplace_ids, region=DEFAULT_REGION,
|
|
| 529 |
str(region or DEFAULT_REGION).lower(), "")
|
| 530 |
# β Pace once per new report. Retrying that same report pays only the shared backoff.
|
| 531 |
_report_pacer["waited"].set(_pace_report_create())
|
| 532 |
-
return _retry(
|
|
|
|
|
|
|
| 533 |
|
| 534 |
|
| 535 |
#: Terminal processing states. `DONE` is the only one that yields a document.
|
|
@@ -640,5 +835,3 @@ def marketplace_participations(fields, timeout=_TIMEOUT):
|
|
| 640 |
got = _request("sellers/v1/marketplaceParticipations", token, region=region, timeout=timeout,
|
| 641 |
sandbox=is_sandbox(fields))
|
| 642 |
return got.get("payload") or []
|
| 643 |
-
|
| 644 |
-
|
|
|
|
| 21 |
signing requests is describing a dead API.
|
| 22 |
Β· https://developer-docs.amazon.com/sp-api/changelog/sp-api-will-no-longer-require-aws-iam-or-aws-signature-version-4
|
| 23 |
|
| 24 |
+
β **READ-ONLY, ENFORCED HERE, WITH EXACTLY TWO NAMED EXCEPTIONS.** `_request` refuses any method but
|
| 25 |
GET before it builds a request. SP-API can create feeds, send buyer messages and adjust listings;
|
| 26 |
none of that is reachable from this module, and the refusal sits below every route and payload so no
|
| 27 |
caller can opt out.
|
| 28 |
|
| 29 |
+
β **The exceptions are `_create_report` and `_competitive_summary_batch`, and both are declared
|
| 30 |
+
rather than hidden.** Asking Amazon to BUILD a report is a POST, and Product Pricing's
|
| 31 |
+
`getCompetitiveSummary` is a POST-wrapped batch of read-only GET subrequests. Both functions have
|
| 32 |
+
their path as a module constant rather than a parameter, so no caller can redirect either one. They
|
| 33 |
+
cannot alter a listing, price, order or inventory level, and cannot send a buyer anything. Full
|
| 34 |
+
reasoning sits on their path constants.
|
|
|
|
|
|
|
| 35 |
|
| 36 |
β **THE BULK PATH IS THE REPORTS API, NEVER PAGE-BY-PAGE.** SP-API is token-bucket rate limited per
|
| 37 |
operation per selling-partner/application pair, and Amazon treats throttling as evidence of a wrong
|
|
|
|
| 73 |
#: https://developer-docs.amazon.com/sp-api/reference/createreport
|
| 74 |
REPORT_SUSTAINED_SECONDS = 60.0
|
| 75 |
|
| 76 |
+
#: Product Pricing v2022-05-01 `getCompetitiveSummary`, official default usage plan.
|
| 77 |
+
#: https://developer-docs.amazon.com/sp-api/reference/getcompetitivesummary
|
| 78 |
+
COMPETITIVE_SUMMARY_RATE = 0.033
|
| 79 |
+
COMPETITIVE_SUMMARY_INTERVAL = 1.0 / COMPETITIVE_SUMMARY_RATE
|
| 80 |
+
COMPETITIVE_SUMMARY_BATCH_SIZE = 20
|
| 81 |
+
|
| 82 |
+
#: AWD v2024-05-09 `listInventory`, official default usage plan and maximum page size.
|
| 83 |
+
#: https://developer-docs.amazon.com/sp-api/reference/listinventory
|
| 84 |
+
AWD_INVENTORY_RATE = 2.0
|
| 85 |
+
AWD_INVENTORY_INTERVAL = 1.0 / AWD_INVENTORY_RATE
|
| 86 |
+
AWD_INVENTORY_PAGE_SIZE = 200
|
| 87 |
+
|
| 88 |
# β One process-wide bucket and lock prevent concurrent syncs from spending the same token.
|
| 89 |
# The contextual wait belongs to the current report_document logger, never another sync's log.
|
| 90 |
_report_pacer = {
|
|
|
|
| 95 |
"waited": contextvars.ContextVar("amazon_report_create_waited", default=0.0),
|
| 96 |
}
|
| 97 |
|
| 98 |
+
#: One process-wide reservation per entity operation. Holding the lock through the short wait keeps
|
| 99 |
+
#: concurrent tenant syncs from spending the same operation token.
|
| 100 |
+
_entity_pacers = {
|
| 101 |
+
"competitive_summary": {"last": None, "lock": threading.Lock()},
|
| 102 |
+
"awd_inventory": {"last": None, "lock": threading.Lock()},
|
| 103 |
+
}
|
| 104 |
+
|
| 105 |
#: β MUST MATCH `core.keychain.ENTRY_TYPES` OR `add_entry` 400s with a message about our data
|
| 106 |
#: structure on a screen about their Amazon account. That was the wave-31 Meta defect and the
|
| 107 |
#: comments in `keychain.py` warn about it in as many words.
|
|
|
|
| 137 |
}
|
| 138 |
DEFAULT_REGION = "na"
|
| 139 |
|
| 140 |
+
#: These paths are constants, not caller input. The outer Product Pricing request is POST, but
|
| 141 |
+
#: every request inside its batch is a fixed GET to the read-only item URI.
|
| 142 |
+
_COMPETITIVE_SUMMARY_BATCH_PATH = (
|
| 143 |
+
"batches/products/pricing/2022-05-01/items/competitiveSummary")
|
| 144 |
+
_COMPETITIVE_SUMMARY_ITEM_URI = "/products/pricing/2022-05-01/items/competitiveSummary"
|
| 145 |
+
_COMPETITIVE_SUMMARY_INCLUDED = ("featuredBuyingOptions", "referencePrices")
|
| 146 |
+
_AWD_INVENTORY_PATH = "awd/2024-05-09/inventory"
|
| 147 |
+
|
| 148 |
|
| 149 |
def is_sandbox(fields):
|
| 150 |
"""Does this stored credential belong to a SANDBOX app? -> bool.
|
|
|
|
| 252 |
},
|
| 253 |
"inventory": {
|
| 254 |
"report": "GET_LEDGER_SUMMARY_VIEW_DATA",
|
| 255 |
+
"entity_read": "AWD v2024-05-09 listInventory (all 200-row pages)",
|
| 256 |
+
"validates_against": "FBA ledger summary vs detail and AWD API items vs stored AWD rows",
|
| 257 |
"role": "Amazon Fulfillment / Amazon Warehousing and Distribution",
|
| 258 |
},
|
| 259 |
# ββ O5 IS CLOSED AND THE ANSWER IS NO (researched 2026-09-04). All 101 report types were
|
|
|
|
| 272 |
"pricing": {
|
| 273 |
"report": "GET_SALES_AND_TRAFFIC_REPORT",
|
| 274 |
"report_covers": "buy box SHARE only; PRICE has no bulk path",
|
| 275 |
+
"price_path": "Product Pricing v2022-05-01 getCompetitiveSummary (batch 20, 0.033/sec)",
|
| 276 |
+
"validates_against": "sales and traffic report totals and competitive ASIN responses",
|
| 277 |
"role": "Pricing + Brand Analytics",
|
| 278 |
},
|
| 279 |
"brand_analytics": {
|
|
|
|
| 331 |
return "<url>"
|
| 332 |
|
| 333 |
|
| 334 |
+
def _retry(attempt_fn, what, *, sleep=time.sleep, rand=random.random, exhausted_fix=None):
|
| 335 |
"""Retry only SP-API throttling; return successes and propagate other refusals unchanged."""
|
| 336 |
for n in range(1, RETRY_MAX_ATTEMPTS + 1):
|
| 337 |
try:
|
|
|
|
| 341 |
# β probe_credential exposes str(e)[:200]; amazon_store.sync stores str(e)[:160].
|
| 342 |
# Throttle, cause AND the complete fix must fit in the first 160 characters;
|
| 343 |
# this head is 152 with the named cap. Operation/URL detail must stay AFTER it.
|
| 344 |
+
fix = exhausted_fix or "sync fewer tables at once and widen the report window"
|
| 345 |
raise AmazonThrottled(
|
| 346 |
f"Amazon throttled (429); {n - 1} retries spent. "
|
| 347 |
"Cause: SP-API limits per operation, app and seller. "
|
| 348 |
+
f"Fix: {fix}. "
|
| 349 |
f"Operation: {_safe(what)}", retry_after=e.retry_after) from None
|
| 350 |
wait = min(RETRY_MAX_SECONDS, RETRY_BASE_SECONDS * 2 ** (n - 1))
|
| 351 |
floor = min(RETRY_MAX_SECONDS, e.retry_after or 0.0)
|
|
|
|
| 375 |
return wait
|
| 376 |
|
| 377 |
|
| 378 |
+
def _pace_entity(name, interval, *, sleep=time.sleep, now=time.monotonic):
|
| 379 |
+
"""Reserve one call for a fixed entity operation under Amazon's default usage plan."""
|
| 380 |
+
pacer = _entity_pacers[name]
|
| 381 |
+
with pacer["lock"]:
|
| 382 |
+
stamp = now()
|
| 383 |
+
last = pacer["last"]
|
| 384 |
+
wait = max(0.0, last + interval - stamp) if last is not None else 0.0
|
| 385 |
+
if wait:
|
| 386 |
+
sleep(wait)
|
| 387 |
+
pacer["last"] = max(stamp + wait, now())
|
| 388 |
+
return wait
|
| 389 |
+
|
| 390 |
+
|
| 391 |
def _post_form(url, data, timeout=_TIMEOUT):
|
| 392 |
body = urllib.parse.urlencode(data).encode("ascii")
|
| 393 |
req = urllib.request.Request(url, data=body, method="POST")
|
|
|
|
| 433 |
|
| 434 |
|
| 435 |
def _request(path, token, region=DEFAULT_REGION, method="GET", timeout=_TIMEOUT,
|
| 436 |
+
sandbox=False, throttle_fix=None, **params):
|
| 437 |
"""One read-only SP-API call. Returns the decoded JSON body.
|
| 438 |
|
| 439 |
β NON-GET IS REFUSED HERE, BEFORE THE REQUEST IS BUILT. There is no flag, no keyword and no
|
|
|
|
| 475 |
detail = ""
|
| 476 |
# 429 is the rate limiter, and standing rule 1 wants it NAMED rather than swallowed.
|
| 477 |
if e.code == 429:
|
| 478 |
+
fix = throttle_fix or "use the Reports API bulk path instead of a slower page loop"
|
| 479 |
raise AmazonThrottled(
|
| 480 |
f"{_safe(url)} was throttled (429). SP-API is token-bucket limited per operation; "
|
| 481 |
+
f"the fix is to {fix}",
|
| 482 |
retry_after=(e.headers or {}).get("Retry-After")) from None
|
| 483 |
raise AmazonError(f"{_safe(url)} answered {e.code}"
|
| 484 |
+ (f": {detail}" if detail else "")) from None
|
|
|
|
| 487 |
|
| 488 |
base = (SANDBOX_ENDPOINTS if sandbox else ENDPOINTS).get(
|
| 489 |
str(region or DEFAULT_REGION).lower(), "")
|
| 490 |
+
return _retry(
|
| 491 |
+
attempt, f"{base}/{str(path).lstrip('/')}",
|
| 492 |
+
exhausted_fix=(throttle_fix
|
| 493 |
+
or "use the Reports API bulk path instead of a slower page loop"))
|
| 494 |
+
|
| 495 |
+
|
| 496 |
+
#: β THE SECOND NON-GET SP-API PATH. `getCompetitiveSummary` is an Amazon-defined POST wrapper,
|
| 497 |
+
#: but every item in its body is a GET to `_COMPETITIVE_SUMMARY_ITEM_URI`. The path, item method,
|
| 498 |
+
#: item URI, included data and 20-item maximum are all fixed below; no caller can turn this into a
|
| 499 |
+
#: write-shaped Product Pricing request.
|
| 500 |
+
def _competitive_summary_batch(token, asins, marketplace_id, region=DEFAULT_REGION,
|
| 501 |
+
timeout=_TIMEOUT, sandbox=False):
|
| 502 |
+
"""Read one fixed `getCompetitiveSummary` batch. -> its response list."""
|
| 503 |
+
items = [str(asin or "").strip() for asin in (asins or ()) if str(asin or "").strip()]
|
| 504 |
+
if not 1 <= len(items) <= COMPETITIVE_SUMMARY_BATCH_SIZE:
|
| 505 |
+
raise AmazonError(
|
| 506 |
+
f"competitive summary needs 1 to {COMPETITIVE_SUMMARY_BATCH_SIZE} ASINs per batch")
|
| 507 |
+
marketplace = str(marketplace_id or "").strip()
|
| 508 |
+
if not marketplace:
|
| 509 |
+
raise AmazonError("competitive summary needs a marketplace_id")
|
| 510 |
+
|
| 511 |
+
body = {"requests": [{
|
| 512 |
+
"asin": asin,
|
| 513 |
+
"marketplaceId": marketplace,
|
| 514 |
+
"includedData": list(_COMPETITIVE_SUMMARY_INCLUDED),
|
| 515 |
+
"method": "GET",
|
| 516 |
+
"uri": _COMPETITIVE_SUMMARY_ITEM_URI,
|
| 517 |
+
} for asin in items]}
|
| 518 |
+
|
| 519 |
+
def attempt():
|
| 520 |
+
base = (SANDBOX_ENDPOINTS if sandbox else ENDPOINTS).get(
|
| 521 |
+
str(region or DEFAULT_REGION).lower())
|
| 522 |
+
if not base:
|
| 523 |
+
raise AmazonError(f"unknown region `{region}`")
|
| 524 |
+
url = f"{base}/{_COMPETITIVE_SUMMARY_BATCH_PATH}"
|
| 525 |
+
req = urllib.request.Request(url, data=json.dumps(body).encode("utf-8"), method="POST")
|
| 526 |
+
req.add_header("x-amz-access-token", token)
|
| 527 |
+
req.add_header("Content-Type", "application/json")
|
| 528 |
+
req.add_header("Accept", "application/json")
|
| 529 |
+
req.add_header("User-Agent", _UA)
|
| 530 |
+
try:
|
| 531 |
+
with urllib.request.urlopen(req, timeout=timeout) as r:
|
| 532 |
+
got = json.loads(r.read().decode("utf-8") or "{}")
|
| 533 |
+
except urllib.error.HTTPError as e:
|
| 534 |
+
detail = ""
|
| 535 |
+
try:
|
| 536 |
+
errs = (json.loads((e.read() or b"").decode("utf-8") or "{}").get("errors") or [])
|
| 537 |
+
detail = str((errs[0] or {}).get("message") or "")[:200] if errs else ""
|
| 538 |
+
except Exception: # noqa: BLE001
|
| 539 |
+
detail = ""
|
| 540 |
+
if e.code == 429:
|
| 541 |
+
raise AmazonThrottled(
|
| 542 |
+
f"{_safe(url)} was throttled (429). getCompetitiveSummary defaults to "
|
| 543 |
+
"0.033 requests/second with a burst of 1; keep 20 ASINs per batch",
|
| 544 |
+
retry_after=(e.headers or {}).get("Retry-After")) from None
|
| 545 |
+
if e.code == 403:
|
| 546 |
+
raise AmazonError(
|
| 547 |
+
f"{_safe(url)} refused getCompetitiveSummary (403). The application needs "
|
| 548 |
+
"the Pricing role") from None
|
| 549 |
+
raise AmazonError(f"{_safe(url)} answered {e.code}"
|
| 550 |
+
+ (f": {detail}" if detail else "")) from None
|
| 551 |
+
except Exception as e: # noqa: BLE001
|
| 552 |
+
raise AmazonError(f"{_safe(url)} unreachable: {type(e).__name__}") from None
|
| 553 |
+
responses = got.get("responses")
|
| 554 |
+
if not isinstance(responses, list):
|
| 555 |
+
raise AmazonError("getCompetitiveSummary returned no response list")
|
| 556 |
+
return responses
|
| 557 |
+
|
| 558 |
+
base = (SANDBOX_ENDPOINTS if sandbox else ENDPOINTS).get(
|
| 559 |
+
str(region or DEFAULT_REGION).lower(), "")
|
| 560 |
+
return _retry(
|
| 561 |
+
attempt, f"{base}/{_COMPETITIVE_SUMMARY_BATCH_PATH}",
|
| 562 |
+
exhausted_fix="retain 20 ASINs per batch and lower the batch request rate")
|
| 563 |
+
|
| 564 |
+
|
| 565 |
+
def competitive_summaries(fields, asins, timeout=_TIMEOUT, *, sleep=time.sleep,
|
| 566 |
+
now=time.monotonic):
|
| 567 |
+
"""Read competitive summaries for every distinct ASIN, in complete fixed-size batches."""
|
| 568 |
+
unique = []
|
| 569 |
+
seen = set()
|
| 570 |
+
for raw in asins or ():
|
| 571 |
+
asin = str(raw or "").strip()
|
| 572 |
+
if asin and asin not in seen:
|
| 573 |
+
unique.append(asin)
|
| 574 |
+
seen.add(asin)
|
| 575 |
+
if not unique:
|
| 576 |
+
return []
|
| 577 |
|
| 578 |
+
f = fields or {}
|
| 579 |
+
region = str(f.get("region") or DEFAULT_REGION)
|
| 580 |
+
marketplace = str(f.get("marketplace_id") or US_MARKETPLACE_ID)
|
| 581 |
+
sandbox = is_sandbox(f)
|
| 582 |
+
token = access_token(f, timeout=timeout)
|
| 583 |
+
token_issued = now()
|
| 584 |
+
out = []
|
| 585 |
+
for offset in range(0, len(unique), COMPETITIVE_SUMMARY_BATCH_SIZE):
|
| 586 |
+
_pace_entity("competitive_summary", COMPETITIVE_SUMMARY_INTERVAL,
|
| 587 |
+
sleep=sleep, now=now)
|
| 588 |
+
if now() - token_issued >= 3000:
|
| 589 |
+
token = access_token(f, timeout=timeout)
|
| 590 |
+
token_issued = now()
|
| 591 |
+
batch = unique[offset:offset + COMPETITIVE_SUMMARY_BATCH_SIZE]
|
| 592 |
+
responses = _competitive_summary_batch(
|
| 593 |
+
token, batch, marketplace, region=region, timeout=timeout, sandbox=sandbox)
|
| 594 |
+
returned = [str(((r or {}).get("body") or {}).get("asin") or "") for r in responses]
|
| 595 |
+
if len(responses) != len(batch) or sorted(returned) != sorted(batch):
|
| 596 |
+
raise AmazonError(
|
| 597 |
+
"getCompetitiveSummary returned an incomplete or mismatched ASIN batch; refusing "
|
| 598 |
+
"to treat a partial response as complete")
|
| 599 |
+
out.extend(responses)
|
| 600 |
+
return out
|
| 601 |
+
|
| 602 |
+
|
| 603 |
+
def awd_inventory(fields, timeout=_TIMEOUT, *, sleep=time.sleep, now=time.monotonic):
|
| 604 |
+
"""Read every AWD inventory page, including through empty pages. -> inventory summaries."""
|
| 605 |
+
f = fields or {}
|
| 606 |
+
region = str(f.get("region") or DEFAULT_REGION)
|
| 607 |
+
sandbox = is_sandbox(f)
|
| 608 |
+
token = access_token(f, timeout=timeout)
|
| 609 |
+
token_issued = now()
|
| 610 |
+
next_token = None
|
| 611 |
+
seen_tokens = set()
|
| 612 |
+
rows = []
|
| 613 |
+
while True:
|
| 614 |
+
_pace_entity("awd_inventory", AWD_INVENTORY_INTERVAL, sleep=sleep, now=now)
|
| 615 |
+
if now() - token_issued >= 3000:
|
| 616 |
+
token = access_token(f, timeout=timeout)
|
| 617 |
+
token_issued = now()
|
| 618 |
+
try:
|
| 619 |
+
got = _request(
|
| 620 |
+
_AWD_INVENTORY_PATH, token, region=region, timeout=timeout, sandbox=sandbox,
|
| 621 |
+
throttle_fix="resume the same listInventory pagination later without dropping pages",
|
| 622 |
+
details="SHOW", maxResults=AWD_INVENTORY_PAGE_SIZE, nextToken=next_token)
|
| 623 |
+
except AmazonError as e:
|
| 624 |
+
if "answered 403" in str(e):
|
| 625 |
+
raise AmazonError(
|
| 626 |
+
f"{_AWD_INVENTORY_PATH} was refused (403). The application needs the Amazon "
|
| 627 |
+
"Warehousing and Distribution role") from None
|
| 628 |
+
raise
|
| 629 |
+
page = got.get("inventory")
|
| 630 |
+
if not isinstance(page, list):
|
| 631 |
+
raise AmazonError("listInventory returned no inventory list")
|
| 632 |
+
rows.extend(page)
|
| 633 |
+
following = str(got.get("nextToken") or "").strip()
|
| 634 |
+
if not following:
|
| 635 |
+
break
|
| 636 |
+
if following in seen_tokens:
|
| 637 |
+
raise AmazonError(
|
| 638 |
+
"listInventory repeated a nextToken; refusing an infinite or duplicate page loop")
|
| 639 |
+
seen_tokens.add(following)
|
| 640 |
+
next_token = following
|
| 641 |
+
return rows
|
| 642 |
|
| 643 |
+
|
| 644 |
+
#: ββ THE REPORT-JOB NON-GET PATH, AND IT IS A CONSTANT RATHER THAN A PARAMETER.
|
| 645 |
#:
|
| 646 |
#: The Reports API bulk path cannot be reached with GET alone: asking Amazon to BUILD a report is
|
| 647 |
#: `POST /reports/2021-06-30/reports`. Standing rule 1 forbids a silent cap on a connector's
|
|
|
|
| 655 |
#: caller-supplied value to smuggle another operation through.
|
| 656 |
#:
|
| 657 |
#: β AND THE BLAST RADIUS IS WORTH STATING PLAINLY, because "read-only" is a promise to the owner
|
| 658 |
+
#: and this is one of two places the HTTP verb bends: a report request creates a JOB, not seller
|
| 659 |
+
#: data. The other is Amazon's POST-wrapped batch of fixed competitive-summary GETs. Neither can
|
| 660 |
+
#: alter a listing, a price, an order or an inventory level, and neither can send a buyer anything.
|
| 661 |
#: The two roles that could do those things were REFUSED at registration (R7).
|
| 662 |
_REPORTS_CREATE_PATH = "reports/2021-06-30/reports"
|
| 663 |
|
|
|
|
| 722 |
str(region or DEFAULT_REGION).lower(), "")
|
| 723 |
# β Pace once per new report. Retrying that same report pays only the shared backoff.
|
| 724 |
_report_pacer["waited"].set(_pace_report_create())
|
| 725 |
+
return _retry(
|
| 726 |
+
attempt, f"{base}/{_REPORTS_CREATE_PATH}",
|
| 727 |
+
exhausted_fix="create fewer reports with wider report windows")
|
| 728 |
|
| 729 |
|
| 730 |
#: Terminal processing states. `DONE` is the only one that yields a document.
|
|
|
|
| 835 |
got = _request("sellers/v1/marketplaceParticipations", token, region=region, timeout=timeout,
|
| 836 |
sandbox=is_sandbox(fields))
|
| 837 |
return got.get("payload") or []
|
|
|
|
|
|
platform/harness/amazon_store.py
CHANGED
|
@@ -1,9 +1,10 @@
|
|
| 1 |
"""amazon_store.py β the Reports-API loader for the tenant mirror, and the only place Amazon rows
|
| 2 |
-
are written.
|
| 3 |
|
| 4 |
python platform/harness/amazon_store.py --sync pull into tenant #0's mirror
|
| 5 |
python platform/harness/amazon_store.py --sync --tenant gtmlab
|
| 6 |
python platform/harness/amazon_store.py --status what is in the mirror now
|
|
|
|
| 7 |
python platform/harness/amazon_store.py --selftest parsers, typing, naming, validate
|
| 8 |
|
| 9 |
β THE SHAPE, and it is deliberately the third instance of one pattern rather than a new one:
|
|
@@ -65,6 +66,14 @@ PREFIX = "amzn_"
|
|
| 65 |
TABLES = ("orders", "order_items", "listings", "settlements", "inventory",
|
| 66 |
"pricing", "brand_analytics")
|
| 67 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 68 |
#: Mirror table names. The `ut_amzn_*` grids are spawned from these by `amazon_relational`.
|
| 69 |
def table_name(table):
|
| 70 |
"""`orders` -> `amzn_orders`. One place, so a rename cannot half-land."""
|
|
@@ -204,6 +213,70 @@ def _flatten_row(obj, prefix=""):
|
|
| 204 |
return out
|
| 205 |
|
| 206 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 207 |
def merge_columns(pinned, parsed):
|
| 208 |
"""The column set for a `schema_artifact` table: the pin, plus anything the payload really had.
|
| 209 |
|
|
@@ -387,6 +460,188 @@ def credential(tenant_key="royal-imports", rt=None):
|
|
| 387 |
return None
|
| 388 |
|
| 389 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 390 |
def _client():
|
| 391 |
"""The SP-API transport, from the DATA LAYER.
|
| 392 |
|
|
@@ -404,6 +659,28 @@ def _client():
|
|
| 404 |
return amazon_client
|
| 405 |
|
| 406 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 407 |
def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None,
|
| 408 |
max_wait=900):
|
| 409 |
"""Pull every declared table into the tenant's mirror. -> a report dict.
|
|
@@ -440,7 +717,7 @@ def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None
|
|
| 440 |
want = [t for t in (tables or TABLES) if t in TABLES]
|
| 441 |
end = datetime.now(timezone.utc)
|
| 442 |
start = since or (end - timedelta(days=DEFAULT_BACKFILL_DAYS))
|
| 443 |
-
con =
|
| 444 |
report = {"tenant": tenant_key, "tables": {}, "started": _iso(end)}
|
| 445 |
|
| 446 |
for table in want:
|
|
@@ -454,7 +731,8 @@ def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None
|
|
| 454 |
prov = amazon_schema.PROVENANCE.get(table, "measured_at_sync")
|
| 455 |
root = (amazon_schema.ARTIFACTS.get(table) or (None, None))[1]
|
| 456 |
try:
|
| 457 |
-
written, cols = 0, []
|
|
|
|
| 458 |
for w0, w1 in _windows(start, end):
|
| 459 |
text, meta = cx.report_document(
|
| 460 |
fields, rtype, start=_iso(w0), end=_iso(w1), max_wait=max_wait, log=log)
|
|
@@ -493,15 +771,78 @@ def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None
|
|
| 493 |
continue
|
| 494 |
if table == "orders":
|
| 495 |
rows = _dedupe_orders(rows)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 496 |
types = _ensure(con, mirror, cols, rows, declared=declared)
|
| 497 |
-
|
| 498 |
-
|
|
|
|
|
|
|
|
|
|
| 499 |
written += _replace(con, mirror, cols, types, rows)
|
| 500 |
-
|
| 501 |
-
|
| 502 |
-
|
| 503 |
-
|
| 504 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 505 |
except Exception as e: # noqa: BLE001
|
| 506 |
# β ONE TABLE'S REFUSAL IS NOT THE SYNC'S. A 403 on brand_analytics is R5's UNPROVEN
|
| 507 |
# ticket answering itself, and it must not cost the six tables that did work.
|
|
@@ -548,7 +889,7 @@ def validate(tenant_key="royal-imports", rt=None, log=print):
|
|
| 548 |
[[a-check-that-cannot-reach-green-is-broken]].
|
| 549 |
"""
|
| 550 |
rt = rt if rt is not None else _runtime_for(tenant_key)
|
| 551 |
-
con =
|
| 552 |
cx = _client()
|
| 553 |
out = {}
|
| 554 |
for table in TABLES:
|
|
@@ -563,6 +904,29 @@ def validate(tenant_key="royal-imports", rt=None, log=print):
|
|
| 563 |
out[table] = {"state": "unproven", "reason": "table is empty",
|
| 564 |
"reconciles": spec.get("validates_against")}
|
| 565 |
continue
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 566 |
out[table] = {"state": "ok", "rows": n, "reconciles": spec.get("validates_against")}
|
| 567 |
for t, r in out.items():
|
| 568 |
log(f" {t:<16} {r['state']:<10} {r.get('rows', 0):>7,} {r.get('reconciles') or ''}")
|
|
@@ -573,7 +937,7 @@ def status(tenant_key="royal-imports", rt=None):
|
|
| 573 |
"""What is in the mirror now. Needs no credential and no network."""
|
| 574 |
rt = rt if rt is not None else _runtime_for(tenant_key)
|
| 575 |
try:
|
| 576 |
-
con =
|
| 577 |
except Exception as e: # noqa: BLE001
|
| 578 |
return {"error": f"{type(e).__name__}: {e}"}
|
| 579 |
out = {}
|
|
@@ -586,6 +950,8 @@ def status(tenant_key="royal-imports", rt=None):
|
|
| 586 |
n, cols = None, 0
|
| 587 |
out[table] = {"rows": n, "columns": cols,
|
| 588 |
"provenance": amazon_schema.PROVENANCE.get(table)}
|
|
|
|
|
|
|
| 589 |
return out
|
| 590 |
|
| 591 |
|
|
@@ -710,7 +1076,150 @@ def selftest(log=print):
|
|
| 710 |
ck("NC-AZ15 an empty document yields no columns and no rows",
|
| 711 |
parse_flat_file("") == ([], []) and parse_json_report("", "x") == ([], []))
|
| 712 |
|
| 713 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 714 |
return 0 if not fails else 1
|
| 715 |
|
| 716 |
|
|
@@ -719,21 +1228,34 @@ def main(argv=None):
|
|
| 719 |
ap.add_argument("--sync", action="store_true")
|
| 720 |
ap.add_argument("--status", action="store_true")
|
| 721 |
ap.add_argument("--validate", action="store_true")
|
|
|
|
|
|
|
| 722 |
ap.add_argument("--selftest", action="store_true")
|
| 723 |
-
ap.add_argument("--tenant", default=
|
|
|
|
|
|
|
| 724 |
ap.add_argument("--table", action="append", help="limit to one table (repeatable)")
|
| 725 |
a = ap.parse_args(argv)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 726 |
if a.selftest:
|
| 727 |
return selftest()
|
| 728 |
if a.status:
|
| 729 |
-
print(json.dumps(status(
|
| 730 |
return 0
|
| 731 |
if a.validate:
|
| 732 |
-
validate(
|
| 733 |
return 0
|
| 734 |
if a.sync:
|
| 735 |
t0 = time.time()
|
| 736 |
-
rep = sync(
|
| 737 |
print(json.dumps(rep, indent=2))
|
| 738 |
print(f"\n{time.time() - t0:.1f}s")
|
| 739 |
return 0
|
|
|
|
| 1 |
"""amazon_store.py β the Reports-API loader for the tenant mirror, and the only place Amazon rows
|
| 2 |
+
are written. Reports remain the bulk spine; Pricing and AWD use their exact read-only operations.
|
| 3 |
|
| 4 |
python platform/harness/amazon_store.py --sync pull into tenant #0's mirror
|
| 5 |
python platform/harness/amazon_store.py --sync --tenant gtmlab
|
| 6 |
python platform/harness/amazon_store.py --status what is in the mirror now
|
| 7 |
+
python platform/harness/amazon_store.py --purge --tenant gtmlab --confirm-tenant gtmlab
|
| 8 |
python platform/harness/amazon_store.py --selftest parsers, typing, naming, validate
|
| 9 |
|
| 10 |
β THE SHAPE, and it is deliberately the third instance of one pattern rather than a new one:
|
|
|
|
| 66 |
TABLES = ("orders", "order_items", "listings", "settlements", "inventory",
|
| 67 |
"pricing", "brand_analytics")
|
| 68 |
|
| 69 |
+
#: Durable proof that the two non-report role operations actually ran. A report-backed table can
|
| 70 |
+
#: contain rows while one of these role grants is still missing, so table presence alone is not
|
| 71 |
+
#: evidence that Pricing or AWD was authorized.
|
| 72 |
+
ENTITY_STATE_KEYS = {
|
| 73 |
+
"pricing": "amzn_pricing_competitive_summary",
|
| 74 |
+
"inventory": "amzn_inventory_awd",
|
| 75 |
+
}
|
| 76 |
+
|
| 77 |
#: Mirror table names. The `ut_amzn_*` grids are spawned from these by `amazon_relational`.
|
| 78 |
def table_name(table):
|
| 79 |
"""`orders` -> `amzn_orders`. One place, so a rename cannot half-land."""
|
|
|
|
| 213 |
return out
|
| 214 |
|
| 215 |
|
| 216 |
+
def _competitive_rows(responses):
|
| 217 |
+
"""Amazon batch responses -> complete flattened enrichments keyed by returned ASIN."""
|
| 218 |
+
out = []
|
| 219 |
+
for response in responses or ():
|
| 220 |
+
body = (response or {}).get("body") or {}
|
| 221 |
+
asin = str(body.get("asin") or "").strip()
|
| 222 |
+
if not asin:
|
| 223 |
+
continue
|
| 224 |
+
row = _flatten_row(response or {}, "competitive")
|
| 225 |
+
row["competitive_body_asin"] = asin
|
| 226 |
+
out.append(row)
|
| 227 |
+
return out
|
| 228 |
+
|
| 229 |
+
|
| 230 |
+
def _awd_rows(inventory):
|
| 231 |
+
"""AWD inventory summaries -> flattened rows that remain distinguishable from FBA ledger."""
|
| 232 |
+
rows = []
|
| 233 |
+
for item in inventory or ():
|
| 234 |
+
row = _flatten_row(item or {})
|
| 235 |
+
row["inventory_source"] = "AWD"
|
| 236 |
+
rows.append(row)
|
| 237 |
+
return rows
|
| 238 |
+
|
| 239 |
+
|
| 240 |
+
def _tag_fba_inventory(rows):
|
| 241 |
+
"""Mark report rows so one combined inventory table never conflates FBA with AWD."""
|
| 242 |
+
for row in rows or ():
|
| 243 |
+
row["inventory_source"] = "FBA"
|
| 244 |
+
return rows
|
| 245 |
+
|
| 246 |
+
|
| 247 |
+
def _pricing_asins(rows):
|
| 248 |
+
"""Distinct child ASINs from Sales and Traffic rows, preserving source order."""
|
| 249 |
+
seen, out = set(), []
|
| 250 |
+
for row in rows or ():
|
| 251 |
+
asin = str(row.get("child_asin") or row.get("parent_asin") or "").strip()
|
| 252 |
+
if asin and asin not in seen:
|
| 253 |
+
seen.add(asin)
|
| 254 |
+
out.append(asin)
|
| 255 |
+
return out
|
| 256 |
+
|
| 257 |
+
|
| 258 |
+
def _apply_competitive(con, table, responses):
|
| 259 |
+
"""Add every competitive response field to all matching Sales and Traffic ASIN rows."""
|
| 260 |
+
rows = _competitive_rows(responses)
|
| 261 |
+
if not rows:
|
| 262 |
+
return 0, 0
|
| 263 |
+
cols = sorted({key for row in rows for key in row})
|
| 264 |
+
types = _ensure(con, table, cols, rows)
|
| 265 |
+
assignments = ", ".join(f'"{c}"=?' for c in cols)
|
| 266 |
+
updated = 0
|
| 267 |
+
for row in rows:
|
| 268 |
+
asin = row["competitive_body_asin"]
|
| 269 |
+
values = [_cell(row.get(c), types.get(c, "VARCHAR")) for c in cols]
|
| 270 |
+
result = con.execute(
|
| 271 |
+
f'UPDATE {table} SET {assignments} WHERE child_asin=? OR parent_asin=?',
|
| 272 |
+
values + [asin, asin])
|
| 273 |
+
try:
|
| 274 |
+
updated += int(result.fetchone()[0])
|
| 275 |
+
except Exception: # noqa: BLE001
|
| 276 |
+
pass
|
| 277 |
+
return len(rows), updated
|
| 278 |
+
|
| 279 |
+
|
| 280 |
def merge_columns(pinned, parsed):
|
| 281 |
"""The column set for a `schema_artifact` table: the pin, plus anything the payload really had.
|
| 282 |
|
|
|
|
| 460 |
return None
|
| 461 |
|
| 462 |
|
| 463 |
+
def _tenant_key(value):
|
| 464 |
+
"""A non-empty canonical tenant slug for a destructive or store-binding operation."""
|
| 465 |
+
key = str(value or "").strip().lower()
|
| 466 |
+
if not key:
|
| 467 |
+
raise ValueError("an explicit tenant slug is required")
|
| 468 |
+
return key
|
| 469 |
+
|
| 470 |
+
|
| 471 |
+
def _tenant_mirror_path(tenant_key, rt=None):
|
| 472 |
+
"""Resolve the exact tenant mirror path without opening or creating it."""
|
| 473 |
+
key = _tenant_key(tenant_key)
|
| 474 |
+
if rt is not None:
|
| 475 |
+
runtime_key = str(getattr(rt, "key", "") or "").strip().lower()
|
| 476 |
+
if runtime_key != key:
|
| 477 |
+
raise RuntimeError(
|
| 478 |
+
f"runtime tenant {runtime_key!r} does not match requested tenant {key!r}")
|
| 479 |
+
return Path(rt.datastore_path)
|
| 480 |
+
return Path(datastore.path_for(key))
|
| 481 |
+
|
| 482 |
+
|
| 483 |
+
def _connect_tenant_mirror(tenant_key, rt=None):
|
| 484 |
+
"""Bind this single-tenant process to the requested mirror, then open a writer cursor.
|
| 485 |
+
|
| 486 |
+
`datastore.connect()` deliberately takes no path argument. The safe path switch is
|
| 487 |
+
`datastore.use_path()`, which closes the cached writer and invalidates every read cursor before
|
| 488 |
+
changing the process-wide file. Calling `connect(path)` looks plausible but raises before a
|
| 489 |
+
credentialed Amazon sync can write its first row.
|
| 490 |
+
"""
|
| 491 |
+
path = _tenant_mirror_path(tenant_key, rt)
|
| 492 |
+
if Path(datastore.DB_PATH).resolve() != path.resolve():
|
| 493 |
+
datastore.use_path(path)
|
| 494 |
+
if rt is not None:
|
| 495 |
+
rt.assert_datastore_matches()
|
| 496 |
+
return datastore.connect(), path
|
| 497 |
+
|
| 498 |
+
|
| 499 |
+
def _require_purge_confirmation(tenant_key, confirmation):
|
| 500 |
+
"""Require the operator to repeat the exact canonical tenant slug."""
|
| 501 |
+
key = _tenant_key(tenant_key)
|
| 502 |
+
if str(confirmation or "").strip() != key:
|
| 503 |
+
raise RuntimeError(
|
| 504 |
+
f"refusing permanent deletion: --confirm-tenant must be exactly {key!r}")
|
| 505 |
+
return key
|
| 506 |
+
|
| 507 |
+
|
| 508 |
+
def _assert_amazon_credential_revoked(rt):
|
| 509 |
+
"""Refuse mirror deletion while any Amazon credential metadata still exists.
|
| 510 |
+
|
| 511 |
+
This checks metadata, not decrypted fields. It therefore still catches a credential when the
|
| 512 |
+
keychain encryption key is unavailable, and an unreadable tenant store raises rather than
|
| 513 |
+
masquerading as an empty keychain.
|
| 514 |
+
"""
|
| 515 |
+
from core import keychain as kc
|
| 516 |
+
if kc.first_entry_of_type(rt, "amazon_sp") is not None:
|
| 517 |
+
raise RuntimeError(
|
| 518 |
+
"refusing permanent deletion: this tenant still has a stored Amazon credential")
|
| 519 |
+
return True
|
| 520 |
+
|
| 521 |
+
|
| 522 |
+
def _quoted_identifier(value):
|
| 523 |
+
"""Quote one database-discovered identifier for a DROP statement."""
|
| 524 |
+
return '"' + str(value).replace('"', '""') + '"'
|
| 525 |
+
|
| 526 |
+
|
| 527 |
+
def _amazon_objects(con):
|
| 528 |
+
"""Every live mirror object reserved to Amazon by the `amzn_` naming contract."""
|
| 529 |
+
rows = con.execute(
|
| 530 |
+
"SELECT table_name, table_type FROM information_schema.tables "
|
| 531 |
+
"WHERE table_schema='main'").fetchall()
|
| 532 |
+
out = []
|
| 533 |
+
for name, kind in rows:
|
| 534 |
+
name = str(name)
|
| 535 |
+
if not name.startswith(PREFIX):
|
| 536 |
+
continue
|
| 537 |
+
out.append((name, str(kind or "BASE TABLE").upper()))
|
| 538 |
+
return sorted(out, key=lambda row: ("VIEW" not in row[1], row[0]))
|
| 539 |
+
|
| 540 |
+
|
| 541 |
+
def _sync_state_rows(con):
|
| 542 |
+
"""The complete state table, used to prove non-Amazon markers survive a purge unchanged."""
|
| 543 |
+
return [tuple(row) for row in con.execute(
|
| 544 |
+
"SELECT entity, phase, last_id, cursor_wd, rows, updated_at "
|
| 545 |
+
"FROM _sync_state ORDER BY entity").fetchall()]
|
| 546 |
+
|
| 547 |
+
|
| 548 |
+
def _purge_mirror(con):
|
| 549 |
+
"""Transactionally delete every `amzn_` mirror object and state marker, then read it back.
|
| 550 |
+
|
| 551 |
+
The prefix is the connector's reserved namespace, so this also catches a future Amazon table
|
| 552 |
+
that has not yet been added to `TABLES`. Non-Amazon tables and sync markers are snapshotted and
|
| 553 |
+
compared inside the transaction and again after commit. No caller supplies a table name.
|
| 554 |
+
"""
|
| 555 |
+
objects = _amazon_objects(con)
|
| 556 |
+
before_tables = {str(row[0]) for row in con.execute(
|
| 557 |
+
"SELECT table_name FROM information_schema.tables "
|
| 558 |
+
"WHERE table_schema='main'").fetchall()}
|
| 559 |
+
before_other_tables = sorted(name for name in before_tables if not name.startswith(PREFIX))
|
| 560 |
+
before_state = _sync_state_rows(con)
|
| 561 |
+
amazon_state = [str(row[0]) for row in before_state if str(row[0]).startswith(PREFIX)]
|
| 562 |
+
other_state = [row for row in before_state if not str(row[0]).startswith(PREFIX)]
|
| 563 |
+
row_counts = {}
|
| 564 |
+
for name, _kind in objects:
|
| 565 |
+
row_counts[name] = int(con.execute(
|
| 566 |
+
f"SELECT COUNT(*) FROM {_quoted_identifier(name)}").fetchone()[0])
|
| 567 |
+
|
| 568 |
+
try:
|
| 569 |
+
con.execute("BEGIN TRANSACTION")
|
| 570 |
+
for name, kind in objects:
|
| 571 |
+
noun = "VIEW" if "VIEW" in kind else "TABLE"
|
| 572 |
+
con.execute(f"DROP {noun} {_quoted_identifier(name)}")
|
| 573 |
+
for entity in amazon_state:
|
| 574 |
+
con.execute("DELETE FROM _sync_state WHERE entity=?", [entity])
|
| 575 |
+
|
| 576 |
+
if _amazon_objects(con):
|
| 577 |
+
raise RuntimeError("Amazon mirror objects remain inside the purge transaction")
|
| 578 |
+
inside_other_tables = sorted(str(row[0]) for row in con.execute(
|
| 579 |
+
"SELECT table_name FROM information_schema.tables WHERE table_schema='main'").fetchall()
|
| 580 |
+
if not str(row[0]).startswith(PREFIX))
|
| 581 |
+
if inside_other_tables != before_other_tables:
|
| 582 |
+
raise RuntimeError("a non-Amazon mirror object changed during the purge")
|
| 583 |
+
inside_state = _sync_state_rows(con)
|
| 584 |
+
if inside_state != other_state:
|
| 585 |
+
raise RuntimeError("a non-Amazon sync marker changed during the purge")
|
| 586 |
+
con.execute("COMMIT")
|
| 587 |
+
except Exception:
|
| 588 |
+
try:
|
| 589 |
+
con.execute("ROLLBACK")
|
| 590 |
+
except Exception: # noqa: BLE001
|
| 591 |
+
pass
|
| 592 |
+
raise
|
| 593 |
+
|
| 594 |
+
residue_objects = _amazon_objects(con)
|
| 595 |
+
residue_state = [str(row[0]) for row in _sync_state_rows(con)
|
| 596 |
+
if str(row[0]).startswith(PREFIX)]
|
| 597 |
+
after_other_tables = sorted(str(row[0]) for row in con.execute(
|
| 598 |
+
"SELECT table_name FROM information_schema.tables WHERE table_schema='main'").fetchall()
|
| 599 |
+
if not str(row[0]).startswith(PREFIX))
|
| 600 |
+
after_other_state = [row for row in _sync_state_rows(con)
|
| 601 |
+
if not str(row[0]).startswith(PREFIX)]
|
| 602 |
+
if residue_objects or residue_state:
|
| 603 |
+
raise RuntimeError(
|
| 604 |
+
"Amazon mirror purge committed but independent readback found residue")
|
| 605 |
+
if after_other_tables != before_other_tables or after_other_state != other_state:
|
| 606 |
+
raise RuntimeError(
|
| 607 |
+
"Amazon mirror purge committed but independent readback found collateral change")
|
| 608 |
+
return {
|
| 609 |
+
"tables": [name for name, _kind in objects],
|
| 610 |
+
"rows": sum(row_counts.values()),
|
| 611 |
+
"syncMarkers": len(amazon_state),
|
| 612 |
+
"residueTables": [],
|
| 613 |
+
"residueSyncMarkers": [],
|
| 614 |
+
}
|
| 615 |
+
|
| 616 |
+
|
| 617 |
+
def purge(tenant_key, confirmation, rt=None, log=print):
|
| 618 |
+
"""Permanently remove one tenant's live Amazon mirror after credential revocation.
|
| 619 |
+
|
| 620 |
+
This is an operator-only lifecycle control. It never revokes a credential itself and refuses
|
| 621 |
+
to run until the tenant runtime resolves, the exact tenant slug is repeated, the mirror already
|
| 622 |
+
exists, and every Amazon credential metadata row is gone.
|
| 623 |
+
"""
|
| 624 |
+
key = _require_purge_confirmation(tenant_key, confirmation)
|
| 625 |
+
rt = rt if rt is not None else _runtime_for(key)
|
| 626 |
+
if rt is None:
|
| 627 |
+
raise RuntimeError(
|
| 628 |
+
f"tenant {key!r} could not be resolved; refusing to guess a database target")
|
| 629 |
+
path = _tenant_mirror_path(key, rt)
|
| 630 |
+
if not path.is_file():
|
| 631 |
+
raise RuntimeError(
|
| 632 |
+
f"tenant {key!r} has no existing mirror file; refusing to create one as purge proof")
|
| 633 |
+
_assert_amazon_credential_revoked(rt)
|
| 634 |
+
con, bound_path = _connect_tenant_mirror(key, rt)
|
| 635 |
+
try:
|
| 636 |
+
result = _purge_mirror(con)
|
| 637 |
+
finally:
|
| 638 |
+
con.close()
|
| 639 |
+
report = {"tenant": key, "database": bound_path.name, **result}
|
| 640 |
+
log(f" tenant {key}: permanently deleted {result['rows']:,} Amazon mirror rows from "
|
| 641 |
+
f"{len(result['tables']):,} tables and {result['syncMarkers']:,} sync markers")
|
| 642 |
+
return report
|
| 643 |
+
|
| 644 |
+
|
| 645 |
def _client():
|
| 646 |
"""The SP-API transport, from the DATA LAYER.
|
| 647 |
|
|
|
|
| 659 |
return amazon_client
|
| 660 |
|
| 661 |
|
| 662 |
+
def _record_state(con, key, phase, rows=0, detail=None):
|
| 663 |
+
"""Write `_sync_state` in its declared column order, never by a misleading positional guess."""
|
| 664 |
+
con.execute(
|
| 665 |
+
"INSERT OR REPLACE INTO _sync_state "
|
| 666 |
+
"(entity, phase, last_id, cursor_wd, rows, updated_at) VALUES (?,?,?,?,?,?)",
|
| 667 |
+
[str(key), str(phase), None, (str(detail)[:300] if detail else None), int(rows or 0),
|
| 668 |
+
_iso(datetime.now(timezone.utc))])
|
| 669 |
+
|
| 670 |
+
|
| 671 |
+
def _read_state(con, key):
|
| 672 |
+
"""One durable operation marker, or None when that operation has never completed."""
|
| 673 |
+
try:
|
| 674 |
+
row = con.execute(
|
| 675 |
+
"SELECT phase, rows, updated_at, cursor_wd FROM _sync_state WHERE entity=?",
|
| 676 |
+
[str(key)]).fetchone()
|
| 677 |
+
except Exception: # noqa: BLE001
|
| 678 |
+
return None
|
| 679 |
+
if not row:
|
| 680 |
+
return None
|
| 681 |
+
return {"state": row[0], "rows": row[1], "updated_at": row[2], "detail": row[3]}
|
| 682 |
+
|
| 683 |
+
|
| 684 |
def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None,
|
| 685 |
max_wait=900):
|
| 686 |
"""Pull every declared table into the tenant's mirror. -> a report dict.
|
|
|
|
| 717 |
want = [t for t in (tables or TABLES) if t in TABLES]
|
| 718 |
end = datetime.now(timezone.utc)
|
| 719 |
start = since or (end - timedelta(days=DEFAULT_BACKFILL_DAYS))
|
| 720 |
+
con, _path = _connect_tenant_mirror(tenant_key, rt)
|
| 721 |
report = {"tenant": tenant_key, "tables": {}, "started": _iso(end)}
|
| 722 |
|
| 723 |
for table in want:
|
|
|
|
| 731 |
prov = amazon_schema.PROVENANCE.get(table, "measured_at_sync")
|
| 732 |
root = (amazon_schema.ARTIFACTS.get(table) or (None, None))[1]
|
| 733 |
try:
|
| 734 |
+
written, cols, cleared = 0, [], False
|
| 735 |
+
pricing_asins = []
|
| 736 |
for w0, w1 in _windows(start, end):
|
| 737 |
text, meta = cx.report_document(
|
| 738 |
fields, rtype, start=_iso(w0), end=_iso(w1), max_wait=max_wait, log=log)
|
|
|
|
| 771 |
continue
|
| 772 |
if table == "orders":
|
| 773 |
rows = _dedupe_orders(rows)
|
| 774 |
+
if table == "inventory":
|
| 775 |
+
rows = _tag_fba_inventory(rows)
|
| 776 |
+
if "inventory_source" not in cols:
|
| 777 |
+
cols.append("inventory_source")
|
| 778 |
+
if table == "pricing":
|
| 779 |
+
pricing_asins.extend(_pricing_asins(rows))
|
| 780 |
types = _ensure(con, mirror, cols, rows, declared=declared)
|
| 781 |
+
# Clear once, after the table exists. The old multi-window branch deleted nothing,
|
| 782 |
+
# so every backfill appended a second copy of every historical row.
|
| 783 |
+
if not cleared:
|
| 784 |
+
con.execute(f"DELETE FROM {mirror} WHERE 1=1")
|
| 785 |
+
cleared = True
|
| 786 |
written += _replace(con, mirror, cols, types, rows)
|
| 787 |
+
|
| 788 |
+
operations = {}
|
| 789 |
+
if table == "pricing":
|
| 790 |
+
state_key = ENTITY_STATE_KEYS[table]
|
| 791 |
+
if not pricing_asins:
|
| 792 |
+
_record_state(con, state_key, "unproven", 0,
|
| 793 |
+
"Sales and Traffic returned no ASIN to test")
|
| 794 |
+
operations["getCompetitiveSummary"] = {
|
| 795 |
+
"state": "unproven", "responses": 0}
|
| 796 |
+
log(" pricing: getCompetitiveSummary UNPROVEN, no ASIN was returned")
|
| 797 |
+
else:
|
| 798 |
+
try:
|
| 799 |
+
responses = cx.competitive_summaries(fields, pricing_asins)
|
| 800 |
+
response_rows, updated = _apply_competitive(con, mirror, responses)
|
| 801 |
+
_record_state(con, state_key, "done", response_rows)
|
| 802 |
+
operations["getCompetitiveSummary"] = {
|
| 803 |
+
"state": "done", "responses": response_rows,
|
| 804 |
+
"mirror_rows_enriched": updated}
|
| 805 |
+
log(f" pricing: getCompetitiveSummary {response_rows:,} ASIN responses, "
|
| 806 |
+
f"{updated:,} mirror rows enriched")
|
| 807 |
+
except Exception as e: # noqa: BLE001
|
| 808 |
+
detail = f"{type(e).__name__}: {str(e)[:160]}"
|
| 809 |
+
_record_state(con, state_key, "refused", 0, detail)
|
| 810 |
+
operations["getCompetitiveSummary"] = {
|
| 811 |
+
"state": "refused", "reason": detail}
|
| 812 |
+
log(f" pricing: getCompetitiveSummary REFUSED: {str(e)[:160]}")
|
| 813 |
+
|
| 814 |
+
if table == "inventory":
|
| 815 |
+
state_key = ENTITY_STATE_KEYS[table]
|
| 816 |
+
try:
|
| 817 |
+
inventory = cx.awd_inventory(fields)
|
| 818 |
+
awd_rows = _awd_rows(inventory)
|
| 819 |
+
if awd_rows:
|
| 820 |
+
awd_cols = sorted({c for row in awd_rows for c in row})
|
| 821 |
+
awd_types = _ensure(con, mirror, awd_cols, awd_rows)
|
| 822 |
+
if not cleared:
|
| 823 |
+
con.execute(f"DELETE FROM {mirror} WHERE 1=1")
|
| 824 |
+
cleared = True
|
| 825 |
+
written += _replace(con, mirror, awd_cols, awd_types, awd_rows)
|
| 826 |
+
_record_state(con, state_key, "done", len(awd_rows))
|
| 827 |
+
operations["listInventory"] = {
|
| 828 |
+
"state": "done", "rows": len(awd_rows)}
|
| 829 |
+
log(f" inventory: AWD listInventory {len(awd_rows):,} rows")
|
| 830 |
+
except Exception as e: # noqa: BLE001
|
| 831 |
+
detail = f"{type(e).__name__}: {str(e)[:160]}"
|
| 832 |
+
_record_state(con, state_key, "refused", 0, detail)
|
| 833 |
+
operations["listInventory"] = {"state": "refused", "reason": detail}
|
| 834 |
+
log(f" inventory: AWD listInventory REFUSED: {str(e)[:160]}")
|
| 835 |
+
|
| 836 |
+
try:
|
| 837 |
+
column_count = len(con.execute(f"PRAGMA table_info('{mirror}')").fetchall())
|
| 838 |
+
total_rows = int(con.execute(f"SELECT COUNT(*) FROM {mirror}").fetchone()[0])
|
| 839 |
+
except Exception: # noqa: BLE001
|
| 840 |
+
column_count, total_rows = len(cols), written
|
| 841 |
+
report["tables"][table] = {
|
| 842 |
+
"rows": total_rows, "columns": column_count, "provenance": prov,
|
| 843 |
+
"report": rtype, "operations": operations}
|
| 844 |
+
_record_state(con, key, "done", total_rows)
|
| 845 |
+
log(f" {table:<16} {total_rows:>7,} rows {column_count:>3} cols {prov}")
|
| 846 |
except Exception as e: # noqa: BLE001
|
| 847 |
# β ONE TABLE'S REFUSAL IS NOT THE SYNC'S. A 403 on brand_analytics is R5's UNPROVEN
|
| 848 |
# ticket answering itself, and it must not cost the six tables that did work.
|
|
|
|
| 889 |
[[a-check-that-cannot-reach-green-is-broken]].
|
| 890 |
"""
|
| 891 |
rt = rt if rt is not None else _runtime_for(tenant_key)
|
| 892 |
+
con, _path = _connect_tenant_mirror(tenant_key, rt)
|
| 893 |
cx = _client()
|
| 894 |
out = {}
|
| 895 |
for table in TABLES:
|
|
|
|
| 904 |
out[table] = {"state": "unproven", "reason": "table is empty",
|
| 905 |
"reconciles": spec.get("validates_against")}
|
| 906 |
continue
|
| 907 |
+
if table in ENTITY_STATE_KEYS:
|
| 908 |
+
operation = _read_state(con, ENTITY_STATE_KEYS[table])
|
| 909 |
+
if not operation or operation.get("state") != "done":
|
| 910 |
+
out[table] = {
|
| 911 |
+
"state": "unproven",
|
| 912 |
+
"reason": (operation or {}).get("detail")
|
| 913 |
+
or "required role operation has not completed",
|
| 914 |
+
"reconciles": spec.get("validates_against")}
|
| 915 |
+
continue
|
| 916 |
+
expected = int(operation.get("rows") or 0)
|
| 917 |
+
if table == "inventory":
|
| 918 |
+
measured = int(con.execute(
|
| 919 |
+
f"SELECT COUNT(*) FROM {mirror} WHERE inventory_source='AWD'").fetchone()[0])
|
| 920 |
+
else:
|
| 921 |
+
measured = int(con.execute(
|
| 922 |
+
f"SELECT COUNT(DISTINCT competitive_body_asin) FROM {mirror} "
|
| 923 |
+
"WHERE competitive_body_asin IS NOT NULL").fetchone()[0])
|
| 924 |
+
if measured != expected:
|
| 925 |
+
out[table] = {
|
| 926 |
+
"state": "refused", "rows": n,
|
| 927 |
+
"reason": f"role operation returned {expected} but mirror carries {measured}",
|
| 928 |
+
"reconciles": spec.get("validates_against")}
|
| 929 |
+
continue
|
| 930 |
out[table] = {"state": "ok", "rows": n, "reconciles": spec.get("validates_against")}
|
| 931 |
for t, r in out.items():
|
| 932 |
log(f" {t:<16} {r['state']:<10} {r.get('rows', 0):>7,} {r.get('reconciles') or ''}")
|
|
|
|
| 937 |
"""What is in the mirror now. Needs no credential and no network."""
|
| 938 |
rt = rt if rt is not None else _runtime_for(tenant_key)
|
| 939 |
try:
|
| 940 |
+
con, _path = _connect_tenant_mirror(tenant_key, rt)
|
| 941 |
except Exception as e: # noqa: BLE001
|
| 942 |
return {"error": f"{type(e).__name__}: {e}"}
|
| 943 |
out = {}
|
|
|
|
| 950 |
n, cols = None, 0
|
| 951 |
out[table] = {"rows": n, "columns": cols,
|
| 952 |
"provenance": amazon_schema.PROVENANCE.get(table)}
|
| 953 |
+
if table in ENTITY_STATE_KEYS:
|
| 954 |
+
out[table]["operation"] = _read_state(con, ENTITY_STATE_KEYS[table])
|
| 955 |
return out
|
| 956 |
|
| 957 |
|
|
|
|
| 1076 |
ck("NC-AZ15 an empty document yields no columns and no rows",
|
| 1077 |
parse_flat_file("") == ([], []) and parse_json_report("", "x") == ([], []))
|
| 1078 |
|
| 1079 |
+
# NC-AZ16..18 - both role-specific payloads keep their complete source shape and source label.
|
| 1080 |
+
price_input = [
|
| 1081 |
+
{"child_asin": "B01", "parent_asin": "P01"},
|
| 1082 |
+
{"child_asin": "B01", "parent_asin": "P01"},
|
| 1083 |
+
{"parent_asin": "P02"},
|
| 1084 |
+
]
|
| 1085 |
+
ck("NC-AZ16 pricing ASINs are complete and de-duplicated",
|
| 1086 |
+
_pricing_asins(price_input) == ["B01", "P02"], _pricing_asins(price_input))
|
| 1087 |
+
competitive = [{
|
| 1088 |
+
"status": {"statusCode": 200, "reasonPhrase": "Success"},
|
| 1089 |
+
"body": {"asin": "B01", "marketplaceId": "ATVPDKIKX0DER",
|
| 1090 |
+
"featuredBuyingOptions": [{"listingPrice": {"amount": 12.5}}],
|
| 1091 |
+
"referencePrices": [{"name": "CompetitivePrice", "price": {"amount": 11.0}}]},
|
| 1092 |
+
}]
|
| 1093 |
+
cr = _competitive_rows(competitive)
|
| 1094 |
+
ck("NC-AZ17 competitive summary keeps status and every returned data family",
|
| 1095 |
+
bool(cr) and cr[0].get("competitive_status_status_code") == 200
|
| 1096 |
+
and "competitive_body_featured_buying_options" in cr[0]
|
| 1097 |
+
and "competitive_body_reference_prices" in cr[0], cr)
|
| 1098 |
+
awd = _awd_rows([{"sku": "RI-1", "totalOnhandQuantity": 9,
|
| 1099 |
+
"inventoryDetails": {"replenishmentQuantity": 2}}])
|
| 1100 |
+
ck("NC-AZ18 AWD inventory remains source-labelled and nested fields survive",
|
| 1101 |
+
bool(awd) and awd[0].get("inventory_source") == "AWD"
|
| 1102 |
+
and awd[0].get("inventory_details_replenishment_quantity") == 2, awd)
|
| 1103 |
+
|
| 1104 |
+
import duckdb
|
| 1105 |
+
mem = duckdb.connect(":memory:")
|
| 1106 |
+
mem.execute("CREATE TABLE amzn_pricing (child_asin VARCHAR, parent_asin VARCHAR)")
|
| 1107 |
+
mem.execute("INSERT INTO amzn_pricing VALUES ('B01','P01'),('B01','P01')")
|
| 1108 |
+
response_count, updated = _apply_competitive(mem, "amzn_pricing", competitive)
|
| 1109 |
+
enriched = mem.execute(
|
| 1110 |
+
"SELECT COUNT(*) FROM amzn_pricing WHERE competitive_body_asin='B01'").fetchone()[0]
|
| 1111 |
+
ck("NC-AZ19 competitive summary enriches every matching report row without appending",
|
| 1112 |
+
response_count == 1 and updated == 2 and enriched == 2,
|
| 1113 |
+
(response_count, updated, enriched))
|
| 1114 |
+
|
| 1115 |
+
mem.execute("""CREATE TABLE _sync_state (
|
| 1116 |
+
entity VARCHAR PRIMARY KEY, phase VARCHAR, last_id BIGINT,
|
| 1117 |
+
cursor_wd VARCHAR, rows BIGINT, updated_at VARCHAR)""")
|
| 1118 |
+
_record_state(mem, "amzn_inventory_awd", "done", 7)
|
| 1119 |
+
marker = mem.execute(
|
| 1120 |
+
"SELECT phase, last_id, cursor_wd, rows, updated_at FROM _sync_state").fetchone()
|
| 1121 |
+
ck("NC-AZ20 sync state writes rows and timestamp into their declared columns",
|
| 1122 |
+
marker[0] == "done" and marker[1] is None and marker[2] is None
|
| 1123 |
+
and marker[3] == 7 and bool(marker[4]), marker)
|
| 1124 |
+
|
| 1125 |
+
# NC-AZ21..24 - the destructive path is hermetic, prefix-complete, and collateral-free.
|
| 1126 |
+
mem.execute("CREATE TABLE amzn_orders (id BIGINT)")
|
| 1127 |
+
mem.execute("INSERT INTO amzn_orders VALUES (1), (2)")
|
| 1128 |
+
mem.execute("CREATE TABLE amzn_future_table (id BIGINT)")
|
| 1129 |
+
mem.execute("INSERT INTO amzn_future_table VALUES (3)")
|
| 1130 |
+
mem.execute("CREATE TABLE keep_unrelated (id BIGINT)")
|
| 1131 |
+
mem.execute("INSERT INTO keep_unrelated VALUES (99)")
|
| 1132 |
+
_record_state(mem, "amzn_orders", "done", 2)
|
| 1133 |
+
_record_state(mem, "odoo_orders", "live", 11)
|
| 1134 |
+
purged = _purge_mirror(mem)
|
| 1135 |
+
ck("NC-AZ21 purge removes every amzn_ table, including a future one",
|
| 1136 |
+
purged["tables"] == ["amzn_future_table", "amzn_orders", "amzn_pricing"]
|
| 1137 |
+
and purged["rows"] == 5, purged)
|
| 1138 |
+
ck("NC-AZ22 purge removes every Amazon sync marker",
|
| 1139 |
+
purged["syncMarkers"] == 2
|
| 1140 |
+
and not [r for r in _sync_state_rows(mem) if str(r[0]).startswith("amzn_")],
|
| 1141 |
+
_sync_state_rows(mem))
|
| 1142 |
+
kept = mem.execute("SELECT id FROM keep_unrelated").fetchall()
|
| 1143 |
+
kept_state = mem.execute(
|
| 1144 |
+
"SELECT phase, rows FROM _sync_state WHERE entity='odoo_orders'").fetchone()
|
| 1145 |
+
ck("NC-AZ23 purge preserves unrelated tables and sync markers byte for byte",
|
| 1146 |
+
kept == [(99,)] and kept_state == ("live", 11), (kept, kept_state))
|
| 1147 |
+
second = _purge_mirror(mem)
|
| 1148 |
+
ck("NC-AZ24 purge readback is residue-free and a retry is idempotent",
|
| 1149 |
+
second == {"tables": [], "rows": 0, "syncMarkers": 0,
|
| 1150 |
+
"residueTables": [], "residueSyncMarkers": []}, second)
|
| 1151 |
+
|
| 1152 |
+
# NC-AZ25..26 - neither a typo nor a still-live credential can reach the DROP transaction.
|
| 1153 |
+
refused_confirmation = False
|
| 1154 |
+
try:
|
| 1155 |
+
_require_purge_confirmation("royal-imports", "royal_imports")
|
| 1156 |
+
except RuntimeError:
|
| 1157 |
+
refused_confirmation = True
|
| 1158 |
+
ck("NC-AZ25 purge refuses when the exact tenant confirmation mismatches",
|
| 1159 |
+
refused_confirmation)
|
| 1160 |
+
|
| 1161 |
+
class _FakeRuntime:
|
| 1162 |
+
def __init__(self, entries):
|
| 1163 |
+
self.entries = entries
|
| 1164 |
+
|
| 1165 |
+
def get(self, name):
|
| 1166 |
+
from core import keychain as _kc
|
| 1167 |
+
return {"entries": self.entries} if name == _kc.KEY else None
|
| 1168 |
+
|
| 1169 |
+
absent_ok, present_refused = False, False
|
| 1170 |
+
try:
|
| 1171 |
+
absent_ok = _assert_amazon_credential_revoked(_FakeRuntime({}))
|
| 1172 |
+
_assert_amazon_credential_revoked(_FakeRuntime({
|
| 1173 |
+
"k_probe": {"type": "amazon_sp", "label": "Amazon",
|
| 1174 |
+
"enc": "not-decrypted-by-this-check"},
|
| 1175 |
+
}))
|
| 1176 |
+
except RuntimeError:
|
| 1177 |
+
present_refused = True
|
| 1178 |
+
ck("NC-AZ26 purge refuses while any Amazon credential metadata remains",
|
| 1179 |
+
absent_ok and present_refused, (absent_ok, present_refused))
|
| 1180 |
+
mem.close()
|
| 1181 |
+
|
| 1182 |
+
# NC-AZ27 - drive the public operator wrapper against an isolated file, including its readback.
|
| 1183 |
+
import tempfile
|
| 1184 |
+
prior_path = Path(datastore.DB_PATH)
|
| 1185 |
+
public_ok, public_detail = False, None
|
| 1186 |
+
try:
|
| 1187 |
+
with tempfile.TemporaryDirectory() as tmp:
|
| 1188 |
+
purge_path = Path(tmp) / "qa-purge.duckdb"
|
| 1189 |
+
seed = duckdb.connect(str(purge_path))
|
| 1190 |
+
seed.execute("""CREATE TABLE _sync_state (
|
| 1191 |
+
entity VARCHAR PRIMARY KEY, phase VARCHAR, last_id BIGINT,
|
| 1192 |
+
cursor_wd VARCHAR, rows BIGINT, updated_at VARCHAR)""")
|
| 1193 |
+
seed.execute("CREATE TABLE amzn_orders (id BIGINT)")
|
| 1194 |
+
seed.execute("INSERT INTO amzn_orders VALUES (1)")
|
| 1195 |
+
_record_state(seed, "amzn_orders", "done", 1)
|
| 1196 |
+
seed.close()
|
| 1197 |
+
|
| 1198 |
+
class _PurgeRuntime(_FakeRuntime):
|
| 1199 |
+
key = "qa-purge"
|
| 1200 |
+
datastore_path = purge_path
|
| 1201 |
+
|
| 1202 |
+
def assert_datastore_matches(self):
|
| 1203 |
+
if Path(datastore.DB_PATH).resolve() != purge_path.resolve():
|
| 1204 |
+
raise RuntimeError("wrong test mirror")
|
| 1205 |
+
|
| 1206 |
+
public_detail = purge(
|
| 1207 |
+
"qa-purge", "qa-purge", rt=_PurgeRuntime({}), log=lambda *_a: None)
|
| 1208 |
+
datastore.use_path(prior_path)
|
| 1209 |
+
readback = duckdb.connect(str(purge_path), read_only=True)
|
| 1210 |
+
tables_after = sorted(str(row[0]) for row in readback.execute("SHOW TABLES").fetchall())
|
| 1211 |
+
state_after = readback.execute("SELECT entity FROM _sync_state").fetchall()
|
| 1212 |
+
readback.close()
|
| 1213 |
+
public_ok = (public_detail["tenant"] == "qa-purge"
|
| 1214 |
+
and public_detail["database"] == "qa-purge.duckdb"
|
| 1215 |
+
and public_detail["rows"] == 1
|
| 1216 |
+
and tables_after == ["_sync_state"] and state_after == [])
|
| 1217 |
+
finally:
|
| 1218 |
+
datastore.use_path(prior_path)
|
| 1219 |
+
ck("NC-AZ27 the public purge wrapper deletes and reads back an isolated tenant file",
|
| 1220 |
+
public_ok, public_detail)
|
| 1221 |
+
|
| 1222 |
+
log(f"\n{'GREEN' if not fails else 'RED'}: {27 - len(fails)}/27")
|
| 1223 |
return 0 if not fails else 1
|
| 1224 |
|
| 1225 |
|
|
|
|
| 1228 |
ap.add_argument("--sync", action="store_true")
|
| 1229 |
ap.add_argument("--status", action="store_true")
|
| 1230 |
ap.add_argument("--validate", action="store_true")
|
| 1231 |
+
ap.add_argument("--purge", action="store_true",
|
| 1232 |
+
help="permanently delete every Amazon mirror table after credential revocation")
|
| 1233 |
ap.add_argument("--selftest", action="store_true")
|
| 1234 |
+
ap.add_argument("--tenant", default=None)
|
| 1235 |
+
ap.add_argument("--confirm-tenant", default=None,
|
| 1236 |
+
help="for --purge, repeat the exact tenant slug")
|
| 1237 |
ap.add_argument("--table", action="append", help="limit to one table (repeatable)")
|
| 1238 |
a = ap.parse_args(argv)
|
| 1239 |
+
tenant = a.tenant or "royal-imports"
|
| 1240 |
+
if a.purge:
|
| 1241 |
+
if not a.tenant:
|
| 1242 |
+
ap.error("--purge requires an explicit --tenant")
|
| 1243 |
+
if a.table or a.sync or a.status or a.validate or a.selftest:
|
| 1244 |
+
ap.error("--purge is all-or-nothing and cannot be combined with another action")
|
| 1245 |
+
rep = purge(a.tenant, a.confirm_tenant)
|
| 1246 |
+
print(json.dumps(rep, indent=2))
|
| 1247 |
+
return 0
|
| 1248 |
if a.selftest:
|
| 1249 |
return selftest()
|
| 1250 |
if a.status:
|
| 1251 |
+
print(json.dumps(status(tenant), indent=2))
|
| 1252 |
return 0
|
| 1253 |
if a.validate:
|
| 1254 |
+
validate(tenant)
|
| 1255 |
return 0
|
| 1256 |
if a.sync:
|
| 1257 |
t0 = time.time()
|
| 1258 |
+
rep = sync(tenant, tables=a.table)
|
| 1259 |
print(json.dumps(rep, indent=2))
|
| 1260 |
print(f"\n{time.time() - t0:.1f}s")
|
| 1261 |
return 0
|
requirements.txt
CHANGED
|
@@ -15,8 +15,9 @@
|
|
| 15 |
# wave 21 (C5) as a LAZY dep of routes_uploads.py β the boot proof above still holds because the
|
| 16 |
# import lives inside the handler, and the gate now asserts this line exists here.
|
| 17 |
fastapi>=0.139
|
|
|
|
| 18 |
uvicorn[standard]>=0.30
|
| 19 |
-
python-dotenv>=1.
|
| 20 |
pandas>=2.0
|
| 21 |
requests>=2.28
|
| 22 |
huggingface_hub>=0.20
|
|
@@ -44,7 +45,7 @@ openpyxl>=3.1
|
|
| 44 |
# python-multipart AT IMPORT TIME for any route declaring File(...)/Form(...). Every local gate
|
| 45 |
# was green because the dev box happens to have it for unrelated reasons β the container
|
| 46 |
# installs exactly this file. The environment-parity twin of the streamlit lesson above.
|
| 47 |
-
python-multipart>=0.0.
|
| 48 |
# β WAVE 37 (D-346, answering `ASK E-1`) β THE OFFICIAL ANTHROPIC SDK. `ai_review.py` called the
|
| 49 |
# Messages API over raw `requests`, which meant re-implementing retries, streaming and error
|
| 50 |
# taxonomy by hand and drifting from them silently. E ships the SDK behind a LAZY import with the
|
|
|
|
| 15 |
# wave 21 (C5) as a LAZY dep of routes_uploads.py β the boot proof above still holds because the
|
| 16 |
# import lives inside the handler, and the gate now asserts this line exists here.
|
| 17 |
fastapi>=0.139
|
| 18 |
+
starlette>=1.3.1
|
| 19 |
uvicorn[standard]>=0.30
|
| 20 |
+
python-dotenv>=1.2.2
|
| 21 |
pandas>=2.0
|
| 22 |
requests>=2.28
|
| 23 |
huggingface_hub>=0.20
|
|
|
|
| 45 |
# python-multipart AT IMPORT TIME for any route declaring File(...)/Form(...). Every local gate
|
| 46 |
# was green because the dev box happens to have it for unrelated reasons β the container
|
| 47 |
# installs exactly this file. The environment-parity twin of the streamlit lesson above.
|
| 48 |
+
python-multipart>=0.0.31
|
| 49 |
# β WAVE 37 (D-346, answering `ASK E-1`) β THE OFFICIAL ANTHROPIC SDK. `ai_review.py` called the
|
| 50 |
# Messages API over raw `requests`, which meant re-implementing retries, streaming and error
|
| 51 |
# taxonomy by hand and drifting from them silently. E ships the SDK behind a LAZY import with the
|