fsanyoto commited on
Commit
07bca1c
Β·
verified Β·
1 Parent(s): 4b54047

Deploy AIOS web (React glide grid + FastAPI slice)

Browse files
Dockerfile CHANGED
@@ -20,12 +20,12 @@
20
  # ---- stage 1: build the React bundle ----
21
  FROM node:22-slim AS web
22
  WORKDIR /web
23
- # Resolve deps FRESH from the public registry (the local lockfile pinned a mirror-only lodash
24
- # 4.18.1 that isn't on npmjs.org; a fresh install gets the public 4.17.x that glide accepts).
25
- COPY web/package.json ./
26
- RUN npm install --no-audit --no-fund --no-package-lock
27
  COPY web/ ./
28
- RUN rm -f package-lock.json && npm run build # -> /web/dist
29
 
30
  # ---- stage 2: python API serving the bundle ----
31
  FROM python:3.12-slim
 
20
  # ---- stage 1: build the React bundle ----
21
  FROM node:22-slim AS web
22
  WORKDIR /web
23
+ # The lockfile is the audited supply-chain boundary. It now resolves entirely from the public
24
+ # registry; discarding it here made every image an unreviewed dependency upgrade.
25
+ COPY web/package.json web/package-lock.json ./
26
+ RUN npm ci --no-audit --no-fund --ignore-scripts
27
  COPY web/ ./
28
+ RUN npm run build # -> /web/dist
29
 
30
  # ---- stage 2: python API serving the bundle ----
31
  FROM python:3.12-slim
RELEASES.json CHANGED
@@ -1,5 +1,5 @@
1
  {
2
- "current": "16028c9",
3
  "releases": [
4
  {
5
  "version": "v71",
 
1
  {
2
+ "current": "bcd2ba1",
3
  "releases": [
4
  {
5
  "version": "v71",
VERSION CHANGED
@@ -1 +1 @@
1
- 16028c9
 
1
+ bcd2ba1
api/amazon_relational.py CHANGED
@@ -95,6 +95,27 @@ IDENTITY = {k: tuple(_SPEC[k].identity) for k in SOURCE if k in _SPEC}
95
  LINKS = {k: [(c, t, o) for c, t, o, _cardinality in _SPEC[k].links]
96
  for k in SOURCE if k in _SPEC and _SPEC[k].links}
97
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
98
  #: Columns whose NAME makes them a date whatever they are stored as. Amazon reports ISO-8601.
99
  _DATEISH = ("purchase_date", "last_updated_date", "payments_date", "settlement_start_date",
100
  "settlement_end_date", "deposit_date", "posted_date", "open_date", "date",
@@ -315,6 +336,9 @@ def apply_plan(rt, built, username="amazon", today=None):
315
  for key, label, fields, rows in plans:
316
  written[key] = _odoo._ensure_table_inplace(doc, key, label, fields, rows,
317
  username, stamp)
 
 
 
318
  return doc
319
 
320
  rt.update(_odoo._ut().STORE_KEY, _apply_all, flush="sync")
 
95
  LINKS = {k: [(c, t, o) for c, t, o, _cardinality in _SPEC[k].links]
96
  for k in SOURCE if k in _SPEC and _SPEC[k].links}
97
 
98
+
99
+ def source_attribution(key):
100
+ """Durable origin metadata for one materialised Amazon database.
101
+
102
+ DPP 1.8 permits a database separation strategy or a mechanism that tags and identifies the
103
+ origin of data. The DuckDB mirror uses the reserved ``amzn_*`` namespace already; this stamp
104
+ is the corresponding mechanism in the shared ``user_tables`` document. It is written on
105
+ every refresh so a user-facing label change cannot erase the Amazon origin or the exact raw
106
+ table/report that supplied the rows.
107
+ """
108
+ if key not in SOURCE or key not in _SPEC:
109
+ raise KeyError(f"unknown Amazon database {key!r}")
110
+ return {
111
+ "key": "amazon_sp",
112
+ "origin": "Amazon Services API",
113
+ "rawTable": SOURCE[key],
114
+ "native": _SPEC[key].native,
115
+ "readOnly": True,
116
+ }
117
+
118
+
119
  #: Columns whose NAME makes them a date whatever they are stored as. Amazon reports ISO-8601.
120
  _DATEISH = ("purchase_date", "last_updated_date", "payments_date", "settlement_start_date",
121
  "settlement_end_date", "deposit_date", "posted_date", "open_date", "date",
 
336
  for key, label, fields, rows in plans:
337
  written[key] = _odoo._ensure_table_inplace(doc, key, label, fields, rows,
338
  username, stamp)
339
+ # DPP 1.8 attribution is explicit in the stored definition, not inferred from a
340
+ # mutable display label or from the process having imported this module.
341
+ doc[key]["connector"] = source_attribution(key)
342
  return doc
343
 
344
  rt.update(_odoo._ut().STORE_KEY, _apply_all, flush="sync")
api/connectors_amazon.py CHANGED
@@ -13,7 +13,7 @@ caller that imports it from here still works.
13
  `keychain.register_prober` is the slot; `datastore.set_paused_probe` is the same idiom one layer
14
  down.
15
 
16
- β›” **READ-ONLY.** See the client's header for the promise and for the single named exception to it.
17
  """
18
  from __future__ import annotations
19
 
@@ -29,11 +29,14 @@ if str(_PLATFORM) not in sys.path:
29
  # ⭐ RE-EXPORTED, NOT REDEFINED. One definition of the credential shape, the endpoints, the report
30
  # map and the transport; this module adds the probe and the registration and nothing else.
31
  from harness.amazon_client import ( # noqa: F401,E402
 
32
  CREDENTIAL_FIELDS, DEFAULT_REGION, ENDPOINTS, KEY, LABEL, LWA_TOKEN_URL, OPTIONAL_FIELDS,
33
  REPORT_TYPES, SANDBOX_ENDPOINTS, base_for, is_sandbox,
34
  ROLES_REFUSED, ROLES_TAKEN, US_MARKETPLACE_ID, AmazonError, _create_report,
35
- _download_document, _post_form, _REPORTS_CREATE_PATH, _request, _safe, _TIMEOUT,
36
- access_token, marketplace_participations, report_document,
 
 
37
  )
38
 
39
  #: ⚠ MUST MATCH `core.keychain.ENTRY_TYPES` OR `add_entry` 400s with a message about our data
 
13
  `keychain.register_prober` is the slot; `datastore.set_paused_probe` is the same idiom one layer
14
  down.
15
 
16
+ β›” **READ-ONLY.** See the client's header for the promise and its two fixed POST wrappers.
17
  """
18
  from __future__ import annotations
19
 
 
29
  # ⭐ RE-EXPORTED, NOT REDEFINED. One definition of the credential shape, the endpoints, the report
30
  # map and the transport; this module adds the probe and the registration and nothing else.
31
  from harness.amazon_client import ( # noqa: F401,E402
32
+ AWD_INVENTORY_PAGE_SIZE, COMPETITIVE_SUMMARY_BATCH_SIZE,
33
  CREDENTIAL_FIELDS, DEFAULT_REGION, ENDPOINTS, KEY, LABEL, LWA_TOKEN_URL, OPTIONAL_FIELDS,
34
  REPORT_TYPES, SANDBOX_ENDPOINTS, base_for, is_sandbox,
35
  ROLES_REFUSED, ROLES_TAKEN, US_MARKETPLACE_ID, AmazonError, _create_report,
36
+ _competitive_summary_batch, _COMPETITIVE_SUMMARY_BATCH_PATH,
37
+ _COMPETITIVE_SUMMARY_ITEM_URI, _download_document, _post_form, _REPORTS_CREATE_PATH,
38
+ _request, _safe, _TIMEOUT, _AWD_INVENTORY_PATH, access_token, awd_inventory,
39
+ competitive_summaries, marketplace_participations, report_document,
40
  )
41
 
42
  #: ⚠ MUST MATCH `core.keychain.ENTRY_TYPES` OR `add_entry` 400s with a message about our data
api/deps.py CHANGED
@@ -776,24 +776,16 @@ def _user_for(claims):
776
  """The account named by a verified cookie, or None β€” with the epoch check that makes a
777
  stateless cookie revocable.
778
 
779
- β›” THE RULE THIS FUNCTION MUST OBEY: **it has to accept exactly the identities `users.verify`
780
  issues.** Any divergence produces the worst failure shape there is β€” a login that returns 200
781
- with a cookie and then 401s every request after it. This function got that wrong once (the
782
- emergency-master branch below ran only when NO record existed, while `verify` grants the master
783
- identity even when one does), so it is now written as a deliberate mirror of `verify`'s
784
- structure: try the record, else the master.
785
 
786
- TWO CASES, in `verify`'s own order:
787
  * THE RECORD, if it is readable and satisfies everything a session adds on top of a login:
788
  the account is active, and its CURRENT epoch equals the cookie's β€” otherwise the session
789
  is revoked (a password change or a deactivation bumped it).
790
- * THE EMERGENCY MASTER β€” username 'admin' with APP_PASSWORD configured β€” exactly as
791
- `verify`'s last branch grants it, whether or not a record exists. This is what stops a
792
- store outage, a self-deactivation or a bumped epoch locking the owner out of the product.
793
- ⚠ AND IT MEANS AN `admin` SESSION IS NOT EPOCH-REVOCABLE while APP_PASSWORD is set. That
794
- is not a weakening: whoever holds APP_PASSWORD can simply log in again, so bumping admin's
795
- epoch never revoked them in the first place. Rotating APP_PASSWORD is how you revoke it.
796
- Every OTHER account is fully epoch-revocable, which is asserted in `verify_api.py`.
797
  * anything else β†’ None. An unknown username is NOT admitted just because its signature was
798
  valid: a signature proves the cookie is ours, not that the account still exists.
799
  """
@@ -831,17 +823,6 @@ def _user_for(claims):
831
  if claim_tenant != pub.get("tenant"):
832
  return None
833
  return pub
834
- if (uname == "admin" and os.environ.get("APP_PASSWORD", "")
835
- and claim_tenant == "royal-imports"):
836
- # The emergency master is TENANT #0's break-glass, not the platform's: a synthetic
837
- # all-access identity minted into another tenant's session would be the exact
838
- # cross-tenant widening the rule above closes.
839
- # ⚠ WAVE 40 (R2): the same mark `users.verify`'s master branch carries. The two dicts are
840
- # documented mirrors of each other, so the key rides on both or they are not mirrors.
841
- return {"username": "admin", "name": "Administrator", "role": "admin",
842
- "bus": "all", "modules": "all", "tenant": "royal-imports",
843
- "emergency_master": True,
844
- "epoch": int(claims.get("e") or 0)}
845
  return None
846
 
847
 
@@ -862,7 +843,7 @@ def _user_for(claims):
862
  # HONEST LIMITS, stated rather than discovered later: the map is per-process, so N workers can each
863
  # write once per window (still bounded); a restart re-arms every user's first request; and the
864
  # resolution of `last_active` is therefore "within the last hour", which is exactly what the plane
865
- # renders it as. Bounded like `routes_auth._FAILS` so it cannot grow into a leak on a deployment
866
  # with many accounts.
867
  _ACTIVE_SEEN: "OrderedDict[str, float]" = OrderedDict()
868
  _ACTIVE_EVERY = float(os.environ.get("AIOS_ACTIVE_STAMP_SECONDS") or 3600)
 
776
  """The account named by a verified cookie, or None β€” with the epoch check that makes a
777
  stateless cookie revocable.
778
 
779
+ β›” THE RULE THIS FUNCTION MUST OBEY: **it has to accept exactly the identities the login route
780
  issues.** Any divergence produces the worst failure shape there is β€” a login that returns 200
781
+ with a cookie and then 401s every request after it.
 
 
 
782
 
783
+ TWO CASES:
784
  * THE RECORD, if it is readable and satisfies everything a session adds on top of a login:
785
  the account is active, and its CURRENT epoch equals the cookie's β€” otherwise the session
786
  is revoked (a password change or a deactivation bumped it).
787
+ * THE QA RUNNER, only when both non-production deployment gates remain enabled and its
788
+ password-derived epoch still matches. Live deployments cannot enable it.
 
 
 
 
 
789
  * anything else β†’ None. An unknown username is NOT admitted just because its signature was
790
  valid: a signature proves the cookie is ours, not that the account still exists.
791
  """
 
823
  if claim_tenant != pub.get("tenant"):
824
  return None
825
  return pub
 
 
 
 
 
 
 
 
 
 
 
826
  return None
827
 
828
 
 
843
  # HONEST LIMITS, stated rather than discovered later: the map is per-process, so N workers can each
844
  # write once per window (still bounded); a restart re-arms every user's first request; and the
845
  # resolution of `last_active` is therefore "within the last hour", which is exactly what the plane
846
+ # renders it as. Bounded independently so it cannot grow into a leak on a deployment
847
  # with many accounts.
848
  _ACTIVE_SEEN: "OrderedDict[str, float]" = OrderedDict()
849
  _ACTIVE_EVERY = float(os.environ.get("AIOS_ACTIVE_STAMP_SECONDS") or 3600)
api/routes_admin.py CHANGED
@@ -216,10 +216,12 @@ def _clean_username(v):
216
  return uname
217
 
218
 
219
- def _clean_password(v, username=None, name=None, email=None, record=None):
 
220
  pw = str(v or "")
221
  failures = users.password_failures(pw, username=username, name=name, email=email,
222
- record=record)
 
223
  if failures:
224
  # Name every broken rule so the caller can correct the value without guessing the policy.
225
  raise err(400, "weak_password",
@@ -414,9 +416,8 @@ def update_user(username: str, body: dict = Body(default=None),
414
  is_self = uname == session.uname
415
  role = _clean_role(body["role"]) if "role" in body else None
416
  active = bool(body["active"]) if "active" in body else None
417
- # THE ONE-CLICK LOCKOUT GUARD. APP_PASSWORD remains the real backstop for 'admin', so this is
418
- # not the thing that keeps the product reachable β€” it just stops an administrator removing
419
- # their own access with a toggle and having to go find the master password.
420
  if is_self and role is not None and role != "admin":
421
  raise err(400, "self_demote",
422
  "you cannot remove your own administrator role; ask another admin")
@@ -467,7 +468,8 @@ def set_password(username: str, body: dict = Body(default=None),
467
  # 404, never 403, because "that account exists in another company" is itself a leak.
468
  raise err(404, "no_such_user", f"no account named {uname!r}")
469
  pw = _clean_password((body or {}).get("password"), username=uname,
470
- name=reg[uname].get("name"), email=reg[uname].get("email"), record=reg[uname])
 
471
  before = int(reg[uname].get("epoch") or 0)
472
  try:
473
  # This route just checked strength AND reuse against its record. Spend the history's
 
216
  return uname
217
 
218
 
219
+ def _clean_password(v, username=None, name=None, email=None, record=None,
220
+ allow_early_rotation=False):
221
  pw = str(v or "")
222
  failures = users.password_failures(pw, username=username, name=name, email=email,
223
+ record=record,
224
+ allow_early_rotation=allow_early_rotation)
225
  if failures:
226
  # Name every broken rule so the caller can correct the value without guessing the policy.
227
  raise err(400, "weak_password",
 
416
  is_self = uname == session.uname
417
  role = _clean_role(body["role"]) if "role" in body else None
418
  active = bool(body["active"]) if "active" in body else None
419
+ # THE ONE-CLICK LOCKOUT GUARD. There is deliberately no APP_PASSWORD bypass, so preventing a
420
+ # stored administrator from removing their own access is part of keeping recovery possible.
 
421
  if is_self and role is not None and role != "admin":
422
  raise err(400, "self_demote",
423
  "you cannot remove your own administrator role; ask another admin")
 
468
  # 404, never 403, because "that account exists in another company" is itself a leak.
469
  raise err(404, "no_such_user", f"no account named {uname!r}")
470
  pw = _clean_password((body or {}).get("password"), username=uname,
471
+ name=reg[uname].get("name"), email=reg[uname].get("email"),
472
+ record=reg[uname], allow_early_rotation=True)
473
  before = int(reg[uname].get("epoch") or 0)
474
  try:
475
  # This route just checked strength AND reuse against its record. Spend the history's
api/routes_auth.py CHANGED
@@ -7,11 +7,7 @@ replaces, and it is deliberately the same account store so the migration is a sw
7
  CREDENTIAL check, not of the user model.
8
  """
9
  import base64 as _b64
10
- import hmac
11
- import os
12
  import re as _re
13
- import time
14
- from collections import OrderedDict
15
 
16
  from fastapi import APIRouter, Body, Depends, Request, Response
17
 
@@ -20,49 +16,46 @@ from deps import Session, err, require_session, users, perms
20
 
21
  router = APIRouter(prefix="/api/v1/auth")
22
 
23
- #: THE SERVER OWNS BACKOFF (X5): the login page shows an inline error and never paces itself, so
24
- #: the pacing has to be here or it does not exist. Bounded so it cannot grow into a memory leak.
25
- #: ⚠ HONEST LIMIT: per-PROCESS. Behind several workers an attacker gets this budget per worker,
26
- #: and it resets on deploy. It raises the cost of online guessing against PBKDF2-200k; it is not
27
- #: a substitute for the real thing, which arrives with Authentik (D-1) owning rate limits centrally.
28
- #: ⚠ KEYED BY (TENANT, USERNAME), not by username. Two tenants can each have an 'admin' or a
29
- #: 'boss', and a shared bucket would mean failed logins against tenant B lock the same name out of
30
- #: tenant A β€” a cross-tenant coupling in the one wave whose point is that tenants cannot reach
31
- #: each other. It is also a (weak) existence oracle: a 429 for a name you never tried here would
32
- #: tell you somebody else did.
33
- _FAILS = OrderedDict()
34
- _MAX_TRACKED = 2048
35
- _LOCKOUT_AFTER = 8
36
- _LOCKOUT_SECONDS = 60
37
 
38
 
39
  def _throttled(key, now=None):
40
- now = now or time.time()
41
- hit = _FAILS.get(key)
42
- if not hit:
43
- return 0
44
- count, last = hit
45
- if count < _LOCKOUT_AFTER:
46
  return 0
47
- remaining = int(_LOCKOUT_SECONDS - (now - last))
48
- return remaining if remaining > 0 else 0
 
 
 
49
 
50
 
51
  def _note_failure(key, now=None):
52
- now = now or time.time()
53
- count, last = _FAILS.get(key, (0, 0.0))
54
- # A window that has fully elapsed starts the count again β€” a user who mistypes twice today
55
- # and twice next week is not an attacker.
56
- if now - last > _LOCKOUT_SECONDS:
57
- count = 0
58
- _FAILS[key] = (count + 1, now)
59
- _FAILS.move_to_end(key)
60
- while len(_FAILS) > _MAX_TRACKED:
61
- _FAILS.popitem(last=False)
62
 
63
 
64
  def _clear_failures(key):
65
- _FAILS.pop(key, None)
 
 
 
 
 
 
66
 
67
 
68
  def _public_user(user):
@@ -145,19 +138,18 @@ def login(request: Request, response: Response, body: dict = Body(default=None))
145
  pw = str(body.get("password") or "")
146
  code = str(body.get("mfaCode") or "").strip()
147
 
148
- # Throttle on the IDENTIFIER alone: the tenant is not known until the credential resolves,
149
- # and a per-(tenant, uname) key would let an attacker reset their budget by rotating slugs.
150
- throttle_key = ("*", uname)
151
- wait = _throttled(throttle_key)
152
- if wait:
153
- raise err(429, "too_many_attempts",
154
- f"too many failed attempts; try again in {wait} seconds")
155
-
156
  # W39-T28 / A62. Resolve this synthetic identity FIRST. The normal account registry is
157
  # tenant #0's document on the PostgreSQL path, so even a harmless `verify()` read would make
158
  # a qa-b proof touch the wrong schema. `qa_identity` is inert unless both staging flags and
159
  # APP_PASSWORD are present; every ordinary username falls straight through to `verify()`.
160
  user = users.qa_identity(uname, pw) if (uname and pw) else None
 
 
 
 
 
 
 
161
  if not user:
162
  user = users.verify(uname, pw) if (uname and pw) else None
163
  if not user:
@@ -182,9 +174,9 @@ def login(request: Request, response: Response, body: dict = Body(default=None))
182
 
183
  # W57-T35 / C12: a correct credential may still need rotation. Check BEFORE bumping or
184
  # minting so an expired password gets the change door and no session, without counting a
185
- # successful authentication against the guessing budget. Neither synthetic identity has a
186
- # stored password clock, and the emergency master must keep its store-outage escape path.
187
- if not (user.get("qa_runner") or user.get("emergency_master")):
188
  try:
189
  rec = (users.registry() or {}).get(user["username"])
190
  except Exception:
@@ -200,10 +192,8 @@ def login(request: Request, response: Response, body: dict = Body(default=None))
200
  pass
201
 
202
  # W57-T36 / C12: every STORED account needs its second factor before any cookie is minted.
203
- # The same TWO identities as the epoch bump are exempt: qa_runner has no registry record
204
- # at all; emergency_master must keep the break-glass path that exists for a store outage.
205
- # Neither has a stored factor this route can require without defeating its existing purpose.
206
- if not (user.get("qa_runner") or user.get("emergency_master")):
207
  try:
208
  rec = (users.registry() or {}).get(user["username"])
209
  except Exception:
@@ -233,21 +223,20 @@ def login(request: Request, response: Response, body: dict = Body(default=None))
233
  # the bump; minting from it would issue every cookie in the product at the OLD epoch, which
234
  # `_user_for` refuses on the very next request β€” "200, then 401 forever".
235
  #
236
- # TWO IDENTITIES ARE EXEMPT, both by design:
237
- # * `qa_runner` β€” the synthetic staging identity has no registry record to bump (its epoch is
238
- # derived from APP_PASSWORD), and `verify_api.py` asserts its login path reads and writes
239
- # NOTHING on tenant #0's `users` key, which a bump's read-modify-write would violate.
240
- # * `emergency_master` β€” `deps._user_for` admits that identity whatever the record's epoch
241
- # says, so a bump revokes nothing there and only risks a store write on the break-glass
242
- # path that exists precisely for a store outage.
243
  #
244
  # A FAILED BUMP MUST NOT FAIL THE LOGIN. `store.update` raises `StoreWriteRefused` where the
245
  # deployment may not write this tenant (mapped to 503 in `main.py`) and `RuntimeError` when the
246
  # store is unavailable, so an unguarded bump would turn every sign-in on such a deployment into
247
  # a 503. The sign-in stands at the epoch it can prove; only the sign-out-everywhere half is
248
  # lost, and the log says so rather than leaving it silent.
 
 
 
249
  session_epoch = int(user.get("epoch") or 0)
250
- if not (user.get("qa_runner") or user.get("emergency_master")):
251
  try:
252
  bumped = users.bump_epoch(user["username"])
253
  if bumped is not None:
@@ -260,9 +249,8 @@ def login(request: Request, response: Response, body: dict = Body(default=None))
260
  aios_session.set_cookie(response, request, value)
261
  # Wave 19 (R4): the login stamp. THE RESOLVED username, never the posted identifier β€” the
262
  # person may have typed an email address and `users.verify` resolved it to the registry key;
263
- # stamping what was typed would write onto a key that does not exist. Fail-silent inside
264
- # `touch_login` and a no-op for the emergency-master identity (which has no record to stamp),
265
- # so the one thing this cannot do is turn a good credential into a failed sign-in.
266
  # The synthetic QA identity has no normal account record to stamp. More importantly, a
267
  # registry stamp here would default to tenant #0 and make a qa-b proof touch the wrong schema.
268
  if not user.get("qa_runner"):
@@ -273,9 +261,6 @@ def login(request: Request, response: Response, body: dict = Body(default=None))
273
  import deps as _deps
274
  if not user.get("qa_runner"):
275
  _deps.note_active(user["username"])
276
- # MOVED from password success: only a WHOLE successful login clears the budget. Otherwise
277
- # a known password would reset the counter before every wrong second-factor guess.
278
- _clear_failures(throttle_key)
279
  return {"user": _public_user(user)}
280
 
281
 
@@ -287,16 +272,16 @@ _MFA_ALREADY_ENROLLED = ("Two step sign in is already on for this account. Ask a
287
 
288
  def _mfa_credentials(uname, password):
289
  """The MFA doors share login's credential check and guessing budget, including aliases."""
290
- throttle_key = ("*", uname)
 
 
 
 
291
  wait = _throttled(throttle_key)
292
  if wait:
293
  raise err(429, "too_many_attempts",
294
  f"too many failed attempts. Try again in {wait} seconds")
295
- # Resolve the staging identity without touching tenant #0's store, then REFUSE it: no
296
- # record exists to enrol. The emergency master is refused for the same no-stored-factor rule.
297
- user = users.qa_identity(uname, password) if (uname and password) else None
298
- if not user:
299
- user = users.verify(uname, password) if (uname and password) else None
300
  if not user:
301
  try:
302
  users._hash(password or "x", "00" * 16)
@@ -304,9 +289,7 @@ def _mfa_credentials(uname, password):
304
  pass
305
  _note_failure(throttle_key)
306
  raise err(401, "invalid_credentials", "that username and password do not match")
307
- if user.get("qa_runner") or user.get("emergency_master"):
308
- raise err(400, "no_stored_account", "That sign in cannot enrol a second factor.")
309
- return user
310
 
311
 
312
  def _mfa_record(uname):
@@ -354,6 +337,8 @@ def _mfa_session(request, response, user, rec, tenant, landed, throttle_key):
354
  if landed is None:
355
  # A missing record/no-op write cannot support a cookie or a promise that codes were saved.
356
  raise err(503, "store_unavailable", "the second factor was not saved. Try again")
 
 
357
  value, _claims = aios_session.mint(tenant, user["username"], int(landed))
358
  aios_session.set_cookie(response, request, value)
359
  users.touch_login(user["username"])
@@ -362,7 +347,6 @@ def _mfa_session(request, response, user, rec, tenant, landed, throttle_key):
362
  if not users.password_changed_at(rec):
363
  # This is a successful first sign-in too: preserve T35's legacy clock backfill.
364
  users.stamp_password_age(user["username"])
365
- _clear_failures(throttle_key)
366
  # Responses carrying a new secret or recovery codes must not survive in an HTTP cache.
367
  response.headers["Cache-Control"] = "no-store"
368
 
@@ -376,7 +360,7 @@ def mfa_enroll(response: Response, body: dict = Body(default=None)):
376
  """
377
  body = body or {}
378
  uname = str(body.get("username") or "").strip().lower()
379
- user = _mfa_credentials(uname, str(body.get("password") or ""))
380
  rec = _mfa_record(user["username"])
381
  if users.mfa_enrolled(rec):
382
  raise err(409, "mfa_already_enrolled", _MFA_ALREADY_ENROLLED)
@@ -396,8 +380,7 @@ def mfa_enroll_confirm(request: Request, response: Response, body: dict = Body(d
396
  """
397
  body = body or {}
398
  uname = str(body.get("username") or "").strip().lower()
399
- throttle_key = ("*", uname)
400
- user = _mfa_credentials(uname, str(body.get("password") or ""))
401
  rec = _mfa_record(user["username"])
402
  if users.mfa_enrolled(rec):
403
  raise err(409, "mfa_already_enrolled", _MFA_ALREADY_ENROLLED)
@@ -416,8 +399,8 @@ def mfa_enroll_confirm(request: Request, response: Response, body: dict = Body(d
416
  @router.get("/mfa")
417
  def mfa_status(session: Session = Depends(require_session)):
418
  """The calling account's state only: never a secret, code or hash."""
419
- # These two identities have no stored factor; do not read tenant #0 for the QA runner.
420
- if session.user.get("qa_runner") or session.user.get("emergency_master"):
421
  rec = {}
422
  else:
423
  rec = _mfa_record(session.uname)
@@ -435,8 +418,7 @@ def mfa_recovery_codes(request: Request, response: Response, body: dict = Body(d
435
  The clear codes appear only in this response; the store and all later reads hold only hashes.
436
  """
437
  uname = session.uname
438
- throttle_key = ("*", uname)
439
- user = _mfa_credentials(uname, str((body or {}).get("password") or ""))
440
  # The session supplies the identity. Body username/tenant fields can never choose a target.
441
  rec = _mfa_record(uname)
442
  if not users.mfa_enrolled(rec):
@@ -452,6 +434,7 @@ def mfa_recovery_codes(request: Request, response: Response, body: dict = Body(d
452
  def password_policy():
453
  """An account-independent checklist the sign-in page can show before authentication."""
454
  return {"rules": users.password_rules(), "minLength": users.PASSWORD_MIN_LENGTH,
 
455
  "maxAgeDays": users.PASSWORD_MAX_AGE_DAYS, "historyCount": users.PASSWORD_HISTORY}
456
 
457
 
@@ -462,18 +445,18 @@ def change_password(body: dict = Body(default=None)):
462
  uname = str(body.get("username") or "").strip().lower()
463
  pw = str(body.get("currentPassword") or "")
464
  new_pw = str(body.get("newPassword") or "")
465
- # Share login's identifier bucket: a second unauthenticated door must not buy a second
466
- # password-guessing budget, and failures must not disclose whether an account exists.
467
- throttle_key = ("*", uname)
 
 
 
 
468
  wait = _throttled(throttle_key)
469
  if wait:
470
  raise err(429, "too_many_attempts",
471
  f"too many failed attempts. Try again in {wait} seconds")
472
- # Resolve the staging runner without registry I/O, exactly as login does. Its credential
473
- # authenticates, but no stored password exists here, so reporting a rotation would be false.
474
- user = users.qa_identity(uname, pw) if (uname and pw) else None
475
- if not user:
476
- user = users.verify(uname, pw) if (uname and pw) else None
477
  if not user:
478
  try:
479
  users._hash(pw or "x", "00" * 16)
@@ -481,8 +464,6 @@ def change_password(body: dict = Body(default=None)):
481
  pass
482
  _note_failure(throttle_key)
483
  raise err(401, "invalid_credentials", "that username and password do not match")
484
- if user.get("emergency_master") or user.get("qa_runner"):
485
- raise err(400, "no_stored_password", "This account has no stored password to change here.")
486
  try:
487
  # verify accepts email addresses too. Rotate the RESOLVED registry key, never the typed
488
  # alias, or this door could return 204 without changing any password or revoking sessions.
 
7
  CREDENTIAL check, not of the user model.
8
  """
9
  import base64 as _b64
 
 
10
  import re as _re
 
 
11
 
12
  from fastapi import APIRouter, Body, Depends, Request, Response
13
 
 
16
 
17
  router = APIRouter(prefix="/api/v1/auth")
18
 
19
+ #: THE SERVER OWNS BACKOFF: the login page never paces itself. The budget lives in the shared
20
+ #: tenant-0 auth store, not this process, so worker rotation or a deploy cannot buy ten more tries.
21
+ #: Username and email aliases resolve to one canonical account key; only a SHA-256 digest of that
22
+ #: normalised key is persisted. Any store failure fails the authentication door closed.
23
+ def _throttle_key(identifier):
24
+ try:
25
+ return users.canonical_login_identifier(identifier)
26
+ except Exception:
27
+ raise err(503, "auth_state_unavailable",
28
+ "sign in protection is temporarily unavailable. Try again later")
 
 
 
 
29
 
30
 
31
  def _throttled(key, now=None):
32
+ if key is None: # staging-only synthetic QA identity
 
 
 
 
 
33
  return 0
34
+ try:
35
+ return users.login_lockout_remaining(key, now=now)
36
+ except Exception:
37
+ raise err(503, "auth_state_unavailable",
38
+ "sign in protection is temporarily unavailable. Try again later")
39
 
40
 
41
  def _note_failure(key, now=None):
42
+ if key is None:
43
+ return 0
44
+ try:
45
+ return users.note_login_failure(key, now=now)
46
+ except Exception:
47
+ raise err(503, "auth_state_unavailable",
48
+ "sign in protection is temporarily unavailable. Try again later")
 
 
 
49
 
50
 
51
  def _clear_failures(key):
52
+ if key is None:
53
+ return
54
+ try:
55
+ users.clear_login_failures(key)
56
+ except Exception:
57
+ raise err(503, "auth_state_unavailable",
58
+ "sign in protection is temporarily unavailable. Try again later")
59
 
60
 
61
  def _public_user(user):
 
138
  pw = str(body.get("password") or "")
139
  code = str(body.get("mfaCode") or "").strip()
140
 
 
 
 
 
 
 
 
 
141
  # W39-T28 / A62. Resolve this synthetic identity FIRST. The normal account registry is
142
  # tenant #0's document on the PostgreSQL path, so even a harmless `verify()` read would make
143
  # a qa-b proof touch the wrong schema. `qa_identity` is inert unless both staging flags and
144
  # APP_PASSWORD are present; every ordinary username falls straight through to `verify()`.
145
  user = users.qa_identity(uname, pw) if (uname and pw) else None
146
+ # A correctly proved QA runner is a staging fixture and remains zero-touch against tenant #0.
147
+ # Every other attempt resolves username/email aliases to one durable account budget.
148
+ throttle_key = None if user else _throttle_key(uname)
149
+ wait = _throttled(throttle_key)
150
+ if wait:
151
+ raise err(429, "too_many_attempts",
152
+ f"too many failed attempts β€” try again in {wait} seconds")
153
  if not user:
154
  user = users.verify(uname, pw) if (uname and pw) else None
155
  if not user:
 
174
 
175
  # W57-T35 / C12: a correct credential may still need rotation. Check BEFORE bumping or
176
  # minting so an expired password gets the change door and no session, without counting a
177
+ # successful authentication against the guessing budget. The staging QA fixture has no
178
+ # stored password clock; every production identity does.
179
+ if not user.get("qa_runner"):
180
  try:
181
  rec = (users.registry() or {}).get(user["username"])
182
  except Exception:
 
192
  pass
193
 
194
  # W57-T36 / C12: every STORED account needs its second factor before any cookie is minted.
195
+ # qa_runner is the sole exception and is double-gated out of live deployments.
196
+ if not user.get("qa_runner"):
 
 
197
  try:
198
  rec = (users.registry() or {}).get(user["username"])
199
  except Exception:
 
223
  # the bump; minting from it would issue every cookie in the product at the OLD epoch, which
224
  # `_user_for` refuses on the very next request β€” "200, then 401 forever".
225
  #
226
+ # qa_runner is the sole exception: it has no registry record to bump (its epoch is derived
227
+ # from APP_PASSWORD), and `verify_api.py` asserts its login path reads and writes NOTHING on
228
+ # tenant #0's `users` key. Live deployments cannot enable this double-gated fixture.
 
 
 
 
229
  #
230
  # A FAILED BUMP MUST NOT FAIL THE LOGIN. `store.update` raises `StoreWriteRefused` where the
231
  # deployment may not write this tenant (mapped to 503 in `main.py`) and `RuntimeError` when the
232
  # store is unavailable, so an unguarded bump would turn every sign-in on such a deployment into
233
  # a 503. The sign-in stands at the epoch it can prove; only the sign-out-everywhere half is
234
  # lost, and the log says so rather than leaving it silent.
235
+ # A complete password+factor proof clears the durable budget BEFORE a cookie is minted. If
236
+ # that security write is unavailable, authentication fails closed.
237
+ _clear_failures(throttle_key)
238
  session_epoch = int(user.get("epoch") or 0)
239
+ if not user.get("qa_runner"):
240
  try:
241
  bumped = users.bump_epoch(user["username"])
242
  if bumped is not None:
 
249
  aios_session.set_cookie(response, request, value)
250
  # Wave 19 (R4): the login stamp. THE RESOLVED username, never the posted identifier β€” the
251
  # person may have typed an email address and `users.verify` resolved it to the registry key;
252
+ # stamping what was typed would write onto a key that does not exist. `touch_login` is
253
+ # fail-silent, so it cannot turn a good credential into a failed sign-in.
 
254
  # The synthetic QA identity has no normal account record to stamp. More importantly, a
255
  # registry stamp here would default to tenant #0 and make a qa-b proof touch the wrong schema.
256
  if not user.get("qa_runner"):
 
261
  import deps as _deps
262
  if not user.get("qa_runner"):
263
  _deps.note_active(user["username"])
 
 
 
264
  return {"user": _public_user(user)}
265
 
266
 
 
272
 
273
  def _mfa_credentials(uname, password):
274
  """The MFA doors share login's credential check and guessing budget, including aliases."""
275
+ # The synthetic runner has no stored factor and is refused without touching tenant #0.
276
+ user = users.qa_identity(uname, password) if (uname and password) else None
277
+ if user:
278
+ raise err(400, "no_stored_account", "That sign in cannot enrol a second factor.")
279
+ throttle_key = _throttle_key(uname)
280
  wait = _throttled(throttle_key)
281
  if wait:
282
  raise err(429, "too_many_attempts",
283
  f"too many failed attempts. Try again in {wait} seconds")
284
+ user = users.verify(uname, password) if (uname and password) else None
 
 
 
 
285
  if not user:
286
  try:
287
  users._hash(password or "x", "00" * 16)
 
289
  pass
290
  _note_failure(throttle_key)
291
  raise err(401, "invalid_credentials", "that username and password do not match")
292
+ return user, throttle_key
 
 
293
 
294
 
295
  def _mfa_record(uname):
 
337
  if landed is None:
338
  # A missing record/no-op write cannot support a cookie or a promise that codes were saved.
339
  raise err(503, "store_unavailable", "the second factor was not saved. Try again")
340
+ # Clear only after the whole factor proof and its write succeed, but before minting a cookie.
341
+ _clear_failures(throttle_key)
342
  value, _claims = aios_session.mint(tenant, user["username"], int(landed))
343
  aios_session.set_cookie(response, request, value)
344
  users.touch_login(user["username"])
 
347
  if not users.password_changed_at(rec):
348
  # This is a successful first sign-in too: preserve T35's legacy clock backfill.
349
  users.stamp_password_age(user["username"])
 
350
  # Responses carrying a new secret or recovery codes must not survive in an HTTP cache.
351
  response.headers["Cache-Control"] = "no-store"
352
 
 
360
  """
361
  body = body or {}
362
  uname = str(body.get("username") or "").strip().lower()
363
+ user, _ = _mfa_credentials(uname, str(body.get("password") or ""))
364
  rec = _mfa_record(user["username"])
365
  if users.mfa_enrolled(rec):
366
  raise err(409, "mfa_already_enrolled", _MFA_ALREADY_ENROLLED)
 
380
  """
381
  body = body or {}
382
  uname = str(body.get("username") or "").strip().lower()
383
+ user, throttle_key = _mfa_credentials(uname, str(body.get("password") or ""))
 
384
  rec = _mfa_record(user["username"])
385
  if users.mfa_enrolled(rec):
386
  raise err(409, "mfa_already_enrolled", _MFA_ALREADY_ENROLLED)
 
399
  @router.get("/mfa")
400
  def mfa_status(session: Session = Depends(require_session)):
401
  """The calling account's state only: never a secret, code or hash."""
402
+ # This identity has no stored factor; do not read tenant #0 for the QA runner.
403
+ if session.user.get("qa_runner"):
404
  rec = {}
405
  else:
406
  rec = _mfa_record(session.uname)
 
418
  The clear codes appear only in this response; the store and all later reads hold only hashes.
419
  """
420
  uname = session.uname
421
+ user, throttle_key = _mfa_credentials(uname, str((body or {}).get("password") or ""))
 
422
  # The session supplies the identity. Body username/tenant fields can never choose a target.
423
  rec = _mfa_record(uname)
424
  if not users.mfa_enrolled(rec):
 
434
  def password_policy():
435
  """An account-independent checklist the sign-in page can show before authentication."""
436
  return {"rules": users.password_rules(), "minLength": users.PASSWORD_MIN_LENGTH,
437
+ "minAgeDays": users.PASSWORD_MIN_AGE_DAYS,
438
  "maxAgeDays": users.PASSWORD_MAX_AGE_DAYS, "historyCount": users.PASSWORD_HISTORY}
439
 
440
 
 
445
  uname = str(body.get("username") or "").strip().lower()
446
  pw = str(body.get("currentPassword") or "")
447
  new_pw = str(body.get("newPassword") or "")
448
+ # The staging runner has no stored password and is refused without touching tenant #0.
449
+ user = users.qa_identity(uname, pw) if (uname and pw) else None
450
+ if user:
451
+ raise err(400, "no_stored_password", "This account has no stored password to change here.")
452
+ # Share login's canonical durable bucket: a second unauthenticated door and an email alias
453
+ # must not buy another password-guessing budget.
454
+ throttle_key = _throttle_key(uname)
455
  wait = _throttled(throttle_key)
456
  if wait:
457
  raise err(429, "too_many_attempts",
458
  f"too many failed attempts. Try again in {wait} seconds")
459
+ user = users.verify(uname, pw) if (uname and pw) else None
 
 
 
 
460
  if not user:
461
  try:
462
  users._hash(pw or "x", "00" * 16)
 
464
  pass
465
  _note_failure(throttle_key)
466
  raise err(401, "invalid_credentials", "that username and password do not match")
 
 
467
  try:
468
  # verify accepts email addresses too. Rotate the RESOLVED registry key, never the typed
469
  # alias, or this door could return 204 without changing any password or revoking sessions.
api/routes_keychain.py CHANGED
@@ -784,16 +784,18 @@ def _credential_revocation(row, rt):
784
  def _revocation_residue(entry_type):
785
  """Standing rule 1 requires the cause AND a fix for the mirror this route cannot clear.
786
 
787
- amazon_store.py --help exposes sync/status/validate/selftest, but no deletion flag.
788
- Clearing the app's materialised copy is not evidence that the raw reports disappeared.
 
789
  """
790
  if entry_type != "amazon_sp":
791
  return []
792
  return [{
793
  "where": "The tenant DuckDB mirror",
794
- "why": "The API process opens the mirror read only, so a route cannot clear it.",
795
- "fix": "An operator must delete the tenant's Amazon mirror tables. "
796
- "platform/harness/amazon_store.py has no flag to do this today.",
 
797
  }]
798
 
799
 
 
784
  def _revocation_residue(entry_type):
785
  """Standing rule 1 requires the cause AND a fix for the mirror this route cannot clear.
786
 
787
+ Clearing the app's materialised copy is not evidence that the raw reports disappeared. Mirror
788
+ deletion stays operator-only because it is permanent, exact-tenant work outside the product
789
+ store transaction.
790
  """
791
  if entry_type != "amazon_sp":
792
  return []
793
  return [{
794
  "where": "The tenant DuckDB mirror",
795
+ "why": "Connection removal clears product databases but not the raw Amazon mirror.",
796
+ "fix": "After every Amazon credential is revoked, an operator must run "
797
+ "platform/harness/amazon_store.py --purge --tenant <tenant> "
798
+ "--confirm-tenant <tenant> and retain the residue-free receipt.",
799
  }]
800
 
801
 
api/routes_sheet.py CHANGED
@@ -912,7 +912,8 @@ def _reresolved_for_reader(session: Session, book):
912
  #: here.
913
  if wid and scope_key:
914
  try:
915
- sheets.fill_scope_cache(store, wid, cell_key, scope_key, out["value"], stamp)
 
916
  except Exception: # noqa: BLE001
917
  pass
918
  if not touched:
@@ -2118,7 +2119,7 @@ def compute_cell(workbook_id: str, addr: str, body: dict = Body(default=None),
2118
  _binding_scope_key(session,
2119
  str(binding.get("database") or ""),
2120
  binding),
2121
- resolved["value"], stamp)
2122
  except Exception: # noqa: BLE001
2123
  pass
2124
 
 
912
  #: here.
913
  if wid and scope_key:
914
  try:
915
+ sheets.fill_scope_cache(store, wid, cell_key, scope_key, out["value"], stamp,
916
+ out["database"])
917
  except Exception: # noqa: BLE001
918
  pass
919
  if not touched:
 
2119
  _binding_scope_key(session,
2120
  str(binding.get("database") or ""),
2121
  binding),
2122
+ resolved["value"], stamp, resolved["database"])
2123
  except Exception: # noqa: BLE001
2124
  pass
2125
 
platform/core/auth_google.py CHANGED
@@ -11,8 +11,8 @@ DESIGN (recorded now so the full build has a contract to meet):
11
  core/users.py account via its `email` attribute (set_access(..., email=...)); an
12
  unknown email FAILS CLOSED with "no account for this Google identity" β€” Google login
13
  never creates users, so the per-BU/per-module grant model stays the only door.
14
- - The password gate stays byte-for-byte intact beside it. APP_PASSWORD remains the
15
- bootstrap master; removing password auth is a separate, owner-approved step.
16
 
17
  ENV (placeholders β€” see .env.example additions in docs/GOOGLE_LOGIN_TODO.md):
18
  GOOGLE_OAUTH_CLIENT_ID OAuth client id from Google Cloud console
 
11
  core/users.py account via its `email` attribute (set_access(..., email=...)); an
12
  unknown email FAILS CLOSED with "no account for this Google identity" β€” Google login
13
  never creates users, so the per-BU/per-module grant model stays the only door.
14
+ - The password gate stays beside it. APP_PASSWORD may seed a fresh registry, but it has no
15
+ authentication bypass; removing stored-password auth is a separate, owner-approved step.
16
 
17
  ENV (placeholders β€” see .env.example additions in docs/GOOGLE_LOGIN_TODO.md):
18
  GOOGLE_OAUTH_CLIENT_ID OAuth client id from Google Cloud console
platform/core/sheets.py CHANGED
@@ -69,6 +69,14 @@ WORKBOOK_KEY_PREFIX = "sheet_workbook_"
69
  #: The companion pointer list. See the module note for why a per-document key needs one.
70
  INDEX_KEY = "sheet_workbooks"
71
 
 
 
 
 
 
 
 
 
72
  # Document format version, independent of the optimistic save revision (`rev`).
73
  SHEET_SCHEMA_VERSION = 1
74
  DOCUMENT_MAX_COLS = 16384
@@ -1347,6 +1355,67 @@ def new_workbook(name="", created_by="", workbook_id=""):
1347
  }
1348
 
1349
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1350
  def create_workbook(st, name="", created_by=""):
1351
  """Write a new `sheet_workbook_<id>` document and register it in the index."""
1352
  book = new_workbook(name=name, created_by=created_by)
@@ -1408,11 +1477,21 @@ def put_workbook(st, workbook_id, raw, expect_rev=None):
1408
  "That workbook changed somewhere else, so this save was not applied.")
1409
  book = migrate_workbook(prior)
1410
  # Immutable ownership and server read projections cannot be reassigned by a PUT.
 
 
 
 
 
 
1411
  protected = {"id", "created_by", "created_at", "updated_at", "rev", "refreshed_at",
1412
- "sheets", "name", "refreshedFor", "shares", "role", "permissions"}
 
1413
  book.update({key: value for key, value in incoming.items() if key not in protected})
1414
  book["name"] = name or _text(prior.get("name"), _MAX_NAME) or "Untitled workbook"
1415
  book["sheets"] = sheets
 
 
 
1416
  book["rev"] = int(prior.get("rev") or 0) + 1
1417
  if refreshed_at is not None:
1418
  book["refreshed_at"] = _text(refreshed_at, 120) or None
@@ -1589,6 +1668,7 @@ def patch_cells(st, workbook_id, tab_ref, raw_cells, expect_rev=None):
1589
  if tab["cols"] > DOCUMENT_MAX_COLS or tab["rows"] > DOCUMENT_MAX_ROWS:
1590
  raise SheetRefusal("sheet_extent_out_of_range", "That cell is outside Excel's sheet dimensions.")
1591
  book["sheets"] = tabs
 
1592
  book["rev"] = current + 1
1593
  book["updated_at"] = _now()
1594
  _json_safe(book, "workbook")
@@ -1677,13 +1757,18 @@ def cached_scope_value(st, workbook_id, cell_key, scope_key):
1677
  return {"value": slot.get("value"), "refreshedAt": slot.get("refreshedAt")}
1678
 
1679
 
1680
- def fill_scope_cache(st, workbook_id, cell_key, scope_key, value, refreshed_at):
1681
  """Write ONE scope's slot for ONE cell. Best effort: a bad workbook id is a no-op, not a raise.
1682
 
1683
- β›”β›” THIS IS THE ONLY DOOR THAT WRITES A SLOT, and it is called from exactly one place: a
1684
- reader's own in-memory re-resolve, right after `_resolve_binding` has answered under that
1685
- reader's own scope. `value` and `refreshed_at` are always what THIS server just computed for
1686
- THIS caller - never a client-supplied number, because nothing here parses a request body.
 
 
 
 
 
1687
  """
1688
  try:
1689
  wid = _slug_id(workbook_id)
@@ -1698,6 +1783,7 @@ def fill_scope_cache(st, workbook_id, cell_key, scope_key, value, refreshed_at):
1698
  slots[scope_key] = {"value": value, "refreshedAt": refreshed_at}
1699
  cells[cell_key] = slots
1700
  doc["cells"] = cells
 
1701
  return doc, None
1702
 
1703
  _persist(st, SCOPE_CACHE_KEY_PREFIX + wid, _apply)
 
69
  #: The companion pointer list. See the module note for why a per-document key needs one.
70
  INDEX_KEY = "sheet_workbooks"
71
 
72
+ #: DPP 1.8's durable origin vocabulary for Amazon-derived Sheet documents. The analytical
73
+ #: database key is retained because one workbook may read several Amazon reports; a bare
74
+ #: ``amazon_sp`` tag would identify the connector but lose which materialised source supplied the
75
+ #: values. This deliberately lives in the document layer rather than in ``amazon_relational``:
76
+ #: Sheet persistence is shared core code and must not import the API's connector implementation.
77
+ AMAZON_DATABASE_PREFIX = "ut_amzn_"
78
+ AMAZON_DATA_ORIGIN_KEY = "amazon_sp"
79
+ AMAZON_DATA_ORIGIN_NAME = "Amazon Services API"
80
  # Document format version, independent of the optimistic save revision (`rev`).
81
  SHEET_SCHEMA_VERSION = 1
82
  DOCUMENT_MAX_COLS = 16384
 
1355
  }
1356
 
1357
 
1358
+ def data_origin_for_database(database):
1359
+ """Canonical durable origin tag for a known connector database, or ``None``.
1360
+
1361
+ DPP 1.8 permits separation or a mechanism that tags the source of Amazon Information. A
1362
+ workbook and its row-scope cache are shared documents, so they use the tagging arm: every
1363
+ ``ut_amzn_*`` source is named explicitly and retained after the binding itself is removed.
1364
+
1365
+ The helper is intentionally conservative. It recognises the reserved Amazon database
1366
+ namespace only; it never guesses that an unrelated database is Amazon from a display label.
1367
+ """
1368
+ database = str(database or "").strip()
1369
+ if not database.startswith(AMAZON_DATABASE_PREFIX):
1370
+ return None
1371
+ return {
1372
+ "key": AMAZON_DATA_ORIGIN_KEY,
1373
+ "origin": AMAZON_DATA_ORIGIN_NAME,
1374
+ "database": database,
1375
+ }
1376
+
1377
+
1378
+ def _merge_data_origins(doc, *, sheets=(), databases=()):
1379
+ """Canonicalise and monotonically extend one document's ``dataOrigins`` inventory.
1380
+
1381
+ ``dataOrigins`` is SERVER-MAINTAINED: PUT bodies never supply it directly. Only a prior
1382
+ canonical tag, a stored binding/source, or an explicit database from a server-side resolve can
1383
+ add an origin. Existing origins are retained deliberately, even when a user later deletes the
1384
+ binding, because the workbook can still hold pasted values and formulas derived from it.
1385
+ """
1386
+ found = set()
1387
+ for raw in (doc.get("dataOrigins") or ()) if isinstance(doc, dict) else ():
1388
+ if not isinstance(raw, dict):
1389
+ continue
1390
+ tag = data_origin_for_database(raw.get("database"))
1391
+ if (tag is not None
1392
+ and raw.get("key") == tag["key"]
1393
+ and raw.get("origin") == tag["origin"]):
1394
+ found.add(tag["database"])
1395
+ for database in databases or ():
1396
+ tag = data_origin_for_database(database)
1397
+ if tag is not None:
1398
+ found.add(tag["database"])
1399
+ for tab in sheets or ():
1400
+ if not isinstance(tab, dict):
1401
+ continue
1402
+ for cell in (tab.get("cells") or {}).values():
1403
+ if not isinstance(cell, dict):
1404
+ continue
1405
+ for member in ("binding", "source"):
1406
+ ref = cell.get(member)
1407
+ tag = data_origin_for_database(
1408
+ ref.get("database") if isinstance(ref, dict) else None)
1409
+ if tag is not None:
1410
+ found.add(tag["database"])
1411
+ origins = [data_origin_for_database(database) for database in sorted(found)]
1412
+ if origins:
1413
+ doc["dataOrigins"] = origins
1414
+ else:
1415
+ doc.pop("dataOrigins", None)
1416
+ return origins
1417
+
1418
+
1419
  def create_workbook(st, name="", created_by=""):
1420
  """Write a new `sheet_workbook_<id>` document and register it in the index."""
1421
  book = new_workbook(name=name, created_by=created_by)
 
1477
  "That workbook changed somewhere else, so this save was not applied.")
1478
  book = migrate_workbook(prior)
1479
  # Immutable ownership and server read projections cannot be reassigned by a PUT.
1480
+ # ``dataOrigins`` is SERVER-MAINTAINED (DPP 1.8) and therefore protected: a PUT body may
1481
+ # neither supply it nor clear it. It is listed here because this blanket copy of the
1482
+ # incoming document would otherwise hand the client both halves of the control β€” sending
1483
+ # ``[]`` erases a genuine Amazon origin, and sending a well-formed tag plants one on an
1484
+ # unrelated workbook. The origin list is rebuilt from the PRIOR document plus the cleaned
1485
+ # cells by ``_merge_data_origins`` immediately below.
1486
  protected = {"id", "created_by", "created_at", "updated_at", "rev", "refreshed_at",
1487
+ "sheets", "name", "refreshedFor", "shares", "role", "permissions",
1488
+ "dataOrigins"}
1489
  book.update({key: value for key, value in incoming.items() if key not in protected})
1490
  book["name"] = name or _text(prior.get("name"), _MAX_NAME) or "Untitled workbook"
1491
  book["sheets"] = sheets
1492
+ # DPP 1.8: merge from the PRIOR document and the cleaned cells. ``raw.dataOrigins`` is
1493
+ # ignored, so a client cannot erase or replace the server-maintained attribution list.
1494
+ _merge_data_origins(book, sheets=sheets)
1495
  book["rev"] = int(prior.get("rev") or 0) + 1
1496
  if refreshed_at is not None:
1497
  book["refreshed_at"] = _text(refreshed_at, 120) or None
 
1668
  if tab["cols"] > DOCUMENT_MAX_COLS or tab["rows"] > DOCUMENT_MAX_ROWS:
1669
  raise SheetRefusal("sheet_extent_out_of_range", "That cell is outside Excel's sheet dimensions.")
1670
  book["sheets"] = tabs
1671
+ _merge_data_origins(book, sheets=tabs)
1672
  book["rev"] = current + 1
1673
  book["updated_at"] = _now()
1674
  _json_safe(book, "workbook")
 
1757
  return {"value": slot.get("value"), "refreshedAt": slot.get("refreshedAt")}
1758
 
1759
 
1760
+ def fill_scope_cache(st, workbook_id, cell_key, scope_key, value, refreshed_at, database):
1761
  """Write ONE scope's slot for ONE cell. Best effort: a bad workbook id is a no-op, not a raise.
1762
 
1763
+ β›”β›” THIS IS THE ONLY DOOR THAT WRITES A SLOT, and its callers are server-side resolve paths:
1764
+ a reader's in-memory re-resolve and the one-cell compute door, each immediately after
1765
+ `_resolve_binding` answers under that caller's own scope. `value`, `refreshed_at` and
1766
+ `database` are always what THIS server just computed - never client-supplied, because nothing
1767
+ here parses a request body.
1768
+
1769
+ DPP 1.8: this cache is a SECOND durable document containing the resolved value, not an
1770
+ implementation detail inside the workbook. Its own top-level origin inventory therefore has
1771
+ to name Amazon independently; relying on the opaque scope hash would not identify a source.
1772
  """
1773
  try:
1774
  wid = _slug_id(workbook_id)
 
1783
  slots[scope_key] = {"value": value, "refreshedAt": refreshed_at}
1784
  cells[cell_key] = slots
1785
  doc["cells"] = cells
1786
+ _merge_data_origins(doc, databases=(database,))
1787
  return doc, None
1788
 
1789
  _persist(st, SCOPE_CACHE_KEY_PREFIX + wid, _apply)
platform/core/users.py CHANGED
@@ -1,8 +1,9 @@
1
  """Per-user accounts for the platform, persisted in the HF Dataset store (users.json).
2
 
3
  Passwords are salted + PBKDF2-HMAC-SHA256 (200k iterations) β€” never stored or logged in plaintext.
4
- A bootstrap 'admin' account is seeded from APP_PASSWORD so the owner can always log in and create
5
- users; APP_PASSWORD also works as an emergency master for 'admin' if the registry is unreachable.
 
6
 
7
  Each account carries BU access ('all' or a list of team-ids [5=Fisch, 6=Royal]) which drives
8
  allowed_bus() β€” the basis for per-Business-Unit permissioning (a Royal-only user never sees Fisch).
@@ -10,6 +11,7 @@ allowed_bus() β€” the basis for per-Business-Unit permissioning (a Royal-only us
10
  import os
11
  import hmac
12
  import hashlib
 
13
  import secrets
14
  import re
15
  import base64
@@ -78,12 +80,23 @@ def _hash(pw, salt):
78
 
79
  # ------------------------------------------------------------------ password policy (W57-T35 / C12)
80
  # ONE policy for account creation and both rotation doors. Login only reads the age: checking
81
- # history there would spend six PBKDF2 passes on every sign-in without improving authentication.
82
  PASSWORD_MIN_LENGTH = 12
83
  PASSWORD_MAX_LENGTH = 256 # bound PBKDF2 input, not a claim about strength
84
- PASSWORD_HISTORY = 5
 
85
  PASSWORD_MAX_AGE_DAYS = 90
86
 
 
 
 
 
 
 
 
 
 
 
87
 
88
  class PasswordPolicyError(ValueError):
89
  """Raised with FAILED RULES attached, never a bare message.
@@ -109,6 +122,8 @@ def password_rules():
109
  {'rule': 'identifier',
110
  'message': 'it must not contain the username, the email local part, or any word of '
111
  'the display name that is 3 characters or longer'},
 
 
112
  {'rule': 'reuse',
113
  'message': f'it must not repeat the current password or any of the last '
114
  f'{PASSWORD_HISTORY} passwords'},
@@ -116,7 +131,8 @@ def password_rules():
116
  ]
117
 
118
 
119
- def password_failures(pw, username=None, name=None, email=None, record=None):
 
120
  """Every rule this password breaks, in policy order. Empty list means it passes."""
121
  pw = str(pw or '')
122
  # Case-fold identifiers, but never the credential being hashed. Name punctuation separates
@@ -135,6 +151,20 @@ def password_failures(pw, username=None, name=None, email=None, record=None):
135
  'symbol': not any(not c.isalnum() for c in pw),
136
  'identifier': any(identifier and identifier in folded for identifier in identifiers),
137
  }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
138
  # An oversized value is already refused; do not feed it to PBKDF2 merely to describe reuse.
139
  # Every history entry owns its own salt, so comparing against only today's salt proves nothing.
140
  if record is not None and len(pw) <= PASSWORD_MAX_LENGTH:
@@ -161,13 +191,16 @@ def password_changed_at(record):
161
  return stamp
162
 
163
 
164
- def password_age_days(record):
165
  """Fractional days since the last password change, or None without a readable stamp."""
166
  import datetime as _dt
167
  stamp = password_changed_at(record)
168
  if stamp is None:
169
  return None
170
- return (_dt.datetime.now(_dt.timezone.utc) - _dt.datetime.fromisoformat(stamp)).total_seconds() / 86400
 
 
 
171
 
172
 
173
  def password_expired(record):
@@ -401,14 +434,132 @@ def _record(pw, name, role, bus, active=True, modules='all', agent=None, email=N
401
  return rec
402
 
403
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
404
  def registry():
405
  return store.get('users')
406
 
407
 
408
  def ensure_bootstrap():
409
  """Seed an 'admin' account from APP_PASSWORD ONLY on a truly fresh store (no users file yet).
410
- Idempotent; no-op if the store is unavailable (the app then falls back to the master-password
411
- path in verify()).
 
 
412
 
413
  Critically, this NEVER overwrites an existing registry: it seeds only when store.exists('users')
414
  is definitively False. A transient read failure at startup used to return {} and make this
@@ -426,8 +577,8 @@ def ensure_bootstrap():
426
  master = os.environ.get('APP_PASSWORD', '')
427
  if not master:
428
  return
429
- # Keep the fresh-store break-glass seed exempt from the password policy. Rejecting whatever
430
- # APP_PASSWORD holds here could leave a new deployment with nobody able to sign in.
431
  try:
432
  store.put('users', {'admin': _record(master, 'Administrator', 'admin', 'all')})
433
  except Exception:
@@ -480,8 +631,8 @@ def epoch(username):
480
 
481
  None is NOT 0. 0 is "this account exists and has never been revoked"; None is "no record" β€”
482
  which the session verifier must treat as a reason to refuse, not as a default to compare
483
- against. (The APP_PASSWORD emergency-master admin has no record at all; the verifier handles
484
- that case explicitly rather than inventing an epoch for it here.)
485
  """
486
  username = (username or '').strip().lower()
487
  try:
@@ -533,18 +684,20 @@ def bump_epoch(username):
533
 
534
 
535
  def verify(username, pw):
536
- """Return a public user dict on success, else None. APP_PASSWORD is an emergency master for the
537
- 'admin' login even if the store is unreachable, so the owner is never locked out."""
 
 
 
538
  username = (username or '').strip().lower()
539
  if not username or not pw:
540
  return None
541
- master = os.environ.get('APP_PASSWORD', '')
542
  try:
543
  # read fresh so accounts created moments ago (UI or out-of-band) are recognised at once
544
  reg = store.get('users', fresh=True)
545
  except Exception:
546
  reg = {}
547
- u = reg.get(username)
548
  # β›” KEY FIRST, EMAIL SECOND, AND SINCE WAVE 37 THAT ORDER IS A DECISION RATHER THAN AN ACCIDENT.
549
  # R8 lets a username BE an email address, so a typed string can now match a registry KEY and a
550
  # different account's `email` field at the same time. The key wins, here, by construction: this
@@ -552,40 +705,8 @@ def verify(username, pw):
552
  # to CREATE either collision (`username_shadows_email` / `email_shadows_username`) so the
553
  # ambiguity cannot be introduced through the product; this line is what decides it for any
554
  # record that arrived some other way.
555
- if u is None and '@' in username:
556
- # Wave 18 (R1): the login box takes a username OR an email β€” admin@nurilab.id signs in
557
- # without knowing the slug an admin chose. First case-insensitive email match wins;
558
- # ambiguity is an admin data problem, not a login feature.
559
- for k, r in reg.items():
560
- if isinstance(r, dict) and str(r.get('email') or '').strip().lower() == username:
561
- username, u = k, r
562
- break
563
  if u and u.get('active', True) and hmac.compare_digest(_hash(pw, u['salt']), u['hash']):
564
  return _public(username, u)
565
- # emergency master: admin + APP_PASSWORD always works (covers first run / store outage)
566
- if username == 'admin' and master and hmac.compare_digest(str(pw), master):
567
- # ⚠ CARRY THE RECORD'S CURRENT EPOCH when there is a record to read, so the session cookie
568
- # the API mints from this dict AGREES with the stored account.
569
- #
570
- # This is not what stops the emergency lockout β€” `deps._user_for`'s master fallback does
571
- # that, and a negative control confirmed the lockout is gone with or without this line.
572
- # What it fixes is subtler and is a SCOPE question: a cookie whose epoch disagrees with the
573
- # record falls through to that master fallback, which hands back a SYNTHETIC identity
574
- # (`bus: 'all'`, `modules: 'all'`). An admin whose record narrows either field would
575
- # therefore be silently WIDENED to consolidated, all-module access for the life of that
576
- # session. Matching the epoch means the record branch wins and the account's real scope
577
- # applies, leaving the master fallback as the true last resort it is meant to be.
578
- # 0 when the store is unreachable, which is the case this branch was written for.
579
- #
580
- # ⚠ WAVE 40 (R2) β€” `emergency_master` MARKS THIS DICT SO THE LOGIN CAN SKIP THE EPOCH BUMP.
581
- # Every ordinary sign-in now bumps the account's epoch to sign its other sessions out, but
582
- # this identity is deliberately NOT epoch-revocable: `deps._user_for`'s master fallback
583
- # admits it whatever the record says. A bump here would therefore revoke nothing, while
584
- # forcing a store WRITE on the one code path that exists for a store outage. Internal to
585
- # the server: `routes_auth._public_user` is a whitelist projection and does not name it.
586
- return {'username': 'admin', 'name': 'Administrator', 'role': 'admin', 'bus': 'all',
587
- 'modules': 'all', 'epoch': int((reg.get('admin') or {}).get('epoch') or 0),
588
- 'emergency_master': True}
589
  return None
590
 
591
 
 
1
  """Per-user accounts for the platform, persisted in the HF Dataset store (users.json).
2
 
3
  Passwords are salted + PBKDF2-HMAC-SHA256 (200k iterations) β€” never stored or logged in plaintext.
4
+ A bootstrap 'admin' account is seeded from APP_PASSWORD only when the registry is genuinely empty.
5
+ After that seed, APP_PASSWORD has no bypass semantics: every production sign-in resolves a stored,
6
+ active account and is subject to password age, durable lockout, and MFA at the API boundary.
7
 
8
  Each account carries BU access ('all' or a list of team-ids [5=Fisch, 6=Royal]) which drives
9
  allowed_bus() β€” the basis for per-Business-Unit permissioning (a Royal-only user never sees Fisch).
 
11
  import os
12
  import hmac
13
  import hashlib
14
+ import math
15
  import secrets
16
  import re
17
  import base64
 
80
 
81
  # ------------------------------------------------------------------ password policy (W57-T35 / C12)
82
  # ONE policy for account creation and both rotation doors. Login only reads the age: checking
83
+ # history there would spend eleven PBKDF2 passes on every sign-in without improving authentication.
84
  PASSWORD_MIN_LENGTH = 12
85
  PASSWORD_MAX_LENGTH = 256 # bound PBKDF2 input, not a claim about strength
86
+ PASSWORD_HISTORY = 10
87
+ PASSWORD_MIN_AGE_DAYS = 1
88
  PASSWORD_MAX_AGE_DAYS = 90
89
 
90
+ # Amazon SP-API's current account-control baseline: no more than ten failed attempts, followed by
91
+ # a thirty-minute suspension. This state is a STORE document rather than a module dict so several
92
+ # workers and a deploy see one budget. The live PostgreSQL backend serialises update() with
93
+ # SELECT ... FOR UPDATE; the HF compatibility backend retains its existing sync-update contract.
94
+ LOGIN_LOCKOUT_AFTER = 10
95
+ LOGIN_LOCKOUT_SECONDS = 30 * 60
96
+ LOGIN_LOCKOUT_BUCKET = 'auth_login_failures'
97
+ _LOGIN_LOCKOUT_MAX_TRACKED = 2048
98
+ _LOGIN_LOCKOUT_RETENTION_SECONDS = 7 * 24 * 60 * 60
99
+
100
 
101
  class PasswordPolicyError(ValueError):
102
  """Raised with FAILED RULES attached, never a bare message.
 
122
  {'rule': 'identifier',
123
  'message': 'it must not contain the username, the email local part, or any word of '
124
  'the display name that is 3 characters or longer'},
125
+ {'rule': 'minimum_age',
126
+ 'message': f'it can normally be changed only after {PASSWORD_MIN_AGE_DAYS} day'},
127
  {'rule': 'reuse',
128
  'message': f'it must not repeat the current password or any of the last '
129
  f'{PASSWORD_HISTORY} passwords'},
 
131
  ]
132
 
133
 
134
+ def password_failures(pw, username=None, name=None, email=None, record=None,
135
+ allow_early_rotation=False, now=None):
136
  """Every rule this password breaks, in policy order. Empty list means it passes."""
137
  pw = str(pw or '')
138
  # Case-fold identifiers, but never the credential being hashed. Name punctuation separates
 
151
  'symbol': not any(not c.isalnum() for c in pw),
152
  'identifier': any(identifier and identifier in folded for identifier in identifiers),
153
  }
154
+ # Amazon DPP 1.4.1's one-day minimum age prevents rapid rotations from walking a password out
155
+ # of the ten-entry history. The first activation/legacy rotation has no history and stays
156
+ # available; an administrator-forced reset marks must_change_password and likewise remains an
157
+ # emergency recovery path. The separate admin password route passes allow_early_rotation only
158
+ # after its role and tenant walls, for a direct suspected-compromise reset.
159
+ age = password_age_days(record, now=now) if record is not None else None
160
+ broken['minimum_age'] = (
161
+ record is not None
162
+ and not allow_early_rotation
163
+ and (record or {}).get('must_change_password') is not True
164
+ and bool((record or {}).get('password_history'))
165
+ and age is not None
166
+ and age < PASSWORD_MIN_AGE_DAYS
167
+ )
168
  # An oversized value is already refused; do not feed it to PBKDF2 merely to describe reuse.
169
  # Every history entry owns its own salt, so comparing against only today's salt proves nothing.
170
  if record is not None and len(pw) <= PASSWORD_MAX_LENGTH:
 
191
  return stamp
192
 
193
 
194
+ def password_age_days(record, now=None):
195
  """Fractional days since the last password change, or None without a readable stamp."""
196
  import datetime as _dt
197
  stamp = password_changed_at(record)
198
  if stamp is None:
199
  return None
200
+ current = now or _dt.datetime.now(_dt.timezone.utc)
201
+ if current.utcoffset() is None:
202
+ current = current.replace(tzinfo=_dt.timezone.utc)
203
+ return (current - _dt.datetime.fromisoformat(stamp)).total_seconds() / 86400
204
 
205
 
206
  def password_expired(record):
 
434
  return rec
435
 
436
 
437
+ def _resolve_identifier(reg, identifier):
438
+ """Return ``(canonical_username, record_or_none)`` using login's key-first rule."""
439
+ identifier = (identifier or '').strip().lower()
440
+ record = (reg or {}).get(identifier)
441
+ if record is None and '@' in identifier:
442
+ for username, candidate in (reg or {}).items():
443
+ if (isinstance(candidate, dict)
444
+ and str(candidate.get('email') or '').strip().lower() == identifier):
445
+ return str(username).strip().lower(), candidate
446
+ return identifier, record
447
+
448
+
449
+ def canonical_login_identifier(identifier):
450
+ """Resolve a username/email alias to the one durable lockout bucket for that account.
451
+
452
+ The value is used only as input to a one-way digest before persistence. Unknown identifiers
453
+ retain their normalised spelling, so their responses remain indistinguishable from a bad
454
+ password for a real account.
455
+ """
456
+ reg = store.get('users', fresh=True) or {}
457
+ canonical, _record_or_none = _resolve_identifier(reg, identifier)
458
+ return canonical
459
+
460
+
461
+ def _login_lockout_key(identifier):
462
+ normalised = (identifier or '').strip().lower()
463
+ return hashlib.sha256(('aios-login-v1\0' + normalised).encode('utf-8')).hexdigest()
464
+
465
+
466
+ def login_lockout_remaining(identifier, now=None):
467
+ """Seconds left in the shared account lock, or zero.
468
+
469
+ Store failures deliberately propagate. A login route that cannot read the shared budget must
470
+ fail closed rather than silently grant each worker a fresh set of guesses.
471
+ """
472
+ now = time.time() if now is None else float(now)
473
+ doc = store.get(LOGIN_LOCKOUT_BUCKET, fresh=True) or {}
474
+ hit = doc.get(_login_lockout_key(identifier)) if isinstance(doc, dict) else None
475
+ if not isinstance(hit, dict) or int(hit.get('count') or 0) < LOGIN_LOCKOUT_AFTER:
476
+ return 0
477
+ try:
478
+ remaining = float(hit.get('locked_until') or 0) - now
479
+ except (TypeError, ValueError, OverflowError):
480
+ return 0
481
+ return max(0, int(math.ceil(remaining)))
482
+
483
+
484
+ def note_login_failure(identifier, now=None):
485
+ """Atomically count one failed password/MFA proof and return lock seconds remaining."""
486
+ now = time.time() if now is None else float(now)
487
+ key = _login_lockout_key(identifier)
488
+
489
+ def _note(doc):
490
+ doc = dict(doc) if isinstance(doc, dict) else {}
491
+ # Keep the security document bounded under random-identifier abuse. An active lock is
492
+ # never pruned, even if a malformed/old ``last_failed_at`` accompanies it.
493
+ retained = {}
494
+ for stored_key, value in doc.items():
495
+ if not isinstance(value, dict):
496
+ continue
497
+ try:
498
+ last = float(value.get('last_failed_at') or 0)
499
+ until = float(value.get('locked_until') or 0)
500
+ except (TypeError, ValueError, OverflowError):
501
+ continue
502
+ if until > now or last >= now - _LOGIN_LOCKOUT_RETENTION_SECONDS:
503
+ retained[stored_key] = value
504
+ doc = retained
505
+
506
+ prior = doc.get(key) or {}
507
+ try:
508
+ count = int(prior.get('count') or 0)
509
+ last = float(prior.get('last_failed_at') or 0)
510
+ until = float(prior.get('locked_until') or 0)
511
+ except (TypeError, ValueError, OverflowError):
512
+ count, last, until = 0, 0.0, 0.0
513
+ # Once a lock expires, or an incomplete failure window ages out, the next miss starts a
514
+ # new budget. Attempts rejected while locked never reach this writer and cannot extend it.
515
+ if until <= now and (count >= LOGIN_LOCKOUT_AFTER or now - last >= LOGIN_LOCKOUT_SECONDS):
516
+ count, until = 0, 0.0
517
+ count += 1
518
+ if count >= LOGIN_LOCKOUT_AFTER:
519
+ until = now + LOGIN_LOCKOUT_SECONDS
520
+ doc[key] = {'count': count, 'last_failed_at': now, 'locked_until': until}
521
+
522
+ if len(doc) > _LOGIN_LOCKOUT_MAX_TRACKED:
523
+ # Discard expired/incomplete budgets before active locks. A pathological set of more
524
+ # than 2,048 simultaneous active identifiers still evicts oldest-first: the document
525
+ # stays bounded rather than becoming an attacker-controlled memory allocation.
526
+ oldest = sorted(doc, key=lambda stored_key: (
527
+ float((doc.get(stored_key) or {}).get('locked_until') or 0) > now,
528
+ float((doc.get(stored_key) or {}).get('last_failed_at') or 0)))
529
+ excess = len(doc) - _LOGIN_LOCKOUT_MAX_TRACKED
530
+ for stored_key in [candidate for candidate in oldest if candidate != key][:excess]:
531
+ doc.pop(stored_key, None)
532
+ return doc
533
+
534
+ landed = store.update(LOGIN_LOCKOUT_BUCKET, _note)
535
+ hit = (landed or {}).get(key) if isinstance(landed, dict) else None
536
+ if not isinstance(hit, dict) or int(hit.get('count') or 0) < LOGIN_LOCKOUT_AFTER:
537
+ return 0
538
+ return max(0, int(math.ceil(float(hit.get('locked_until') or 0) - now)))
539
+
540
+
541
+ def clear_login_failures(identifier):
542
+ """Clear the account's shared guessing budget after a complete successful proof."""
543
+ key = _login_lockout_key(identifier)
544
+
545
+ def _clear(doc):
546
+ doc = dict(doc) if isinstance(doc, dict) else {}
547
+ doc.pop(key, None)
548
+ return doc
549
+
550
+ return store.update(LOGIN_LOCKOUT_BUCKET, _clear)
551
+
552
+
553
  def registry():
554
  return store.get('users')
555
 
556
 
557
  def ensure_bootstrap():
558
  """Seed an 'admin' account from APP_PASSWORD ONLY on a truly fresh store (no users file yet).
559
+
560
+ Idempotent and a no-op if the store is unavailable. There is intentionally no authentication
561
+ fallback: after the seed, the stored account is the sole production identity and must enrol
562
+ MFA like every other account.
563
 
564
  Critically, this NEVER overwrites an existing registry: it seeds only when store.exists('users')
565
  is definitively False. A transient read failure at startup used to return {} and make this
 
577
  master = os.environ.get('APP_PASSWORD', '')
578
  if not master:
579
  return
580
+ # Keep the fresh-store bootstrap seed exempt from the password policy. Rejecting whatever
581
+ # APP_PASSWORD holds here could leave a new deployment with nobody able to begin enrolment.
582
  try:
583
  store.put('users', {'admin': _record(master, 'Administrator', 'admin', 'all')})
584
  except Exception:
 
631
 
632
  None is NOT 0. 0 is "this account exists and has never been revoked"; None is "no record" β€”
633
  which the session verifier must treat as a reason to refuse, not as a default to compare
634
+ against. A username with no stored account is always refused; a signed cookie is not an
635
+ identity record.
636
  """
637
  username = (username or '').strip().lower()
638
  try:
 
684
 
685
 
686
  def verify(username, pw):
687
+ """Return a public stored user on success, else ``None``.
688
+
689
+ ``APP_PASSWORD`` may have seeded the first admin record, but it is never consulted here. A
690
+ missing/unreachable registry therefore fails closed instead of minting an MFA-exempt identity.
691
+ """
692
  username = (username or '').strip().lower()
693
  if not username or not pw:
694
  return None
 
695
  try:
696
  # read fresh so accounts created moments ago (UI or out-of-band) are recognised at once
697
  reg = store.get('users', fresh=True)
698
  except Exception:
699
  reg = {}
700
+ username, u = _resolve_identifier(reg, username)
701
  # β›” KEY FIRST, EMAIL SECOND, AND SINCE WAVE 37 THAT ORDER IS A DECISION RATHER THAN AN ACCIDENT.
702
  # R8 lets a username BE an email address, so a typed string can now match a registry KEY and a
703
  # different account's `email` field at the same time. The key wins, here, by construction: this
 
705
  # to CREATE either collision (`username_shadows_email` / `email_shadows_username`) so the
706
  # ambiguity cannot be introduced through the product; this line is what decides it for any
707
  # record that arrived some other way.
 
 
 
 
 
 
 
 
708
  if u and u.get('active', True) and hmac.compare_digest(_hash(pw, u['salt']), u['hash']):
709
  return _public(username, u)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
710
  return None
711
 
712
 
platform/harness/amazon_client.py CHANGED
@@ -21,19 +21,17 @@ disregards AWS SigV4 entirely and authorizes on the LWA access token alone: a pl
21
  signing requests is describing a dead API.
22
  Β· https://developer-docs.amazon.com/sp-api/changelog/sp-api-will-no-longer-require-aws-iam-or-aws-signature-version-4
23
 
24
- β›” **READ-ONLY, ENFORCED HERE, WITH EXACTLY ONE NAMED EXCEPTION.** `_request` refuses any method but
25
  GET before it builds a request. SP-API can create feeds, send buyer messages and adjust listings;
26
  none of that is reachable from this module, and the refusal sits below every route and payload so no
27
  caller can opt out.
28
 
29
- ⚠ **The exception is `_create_report`, and it is declared rather than hidden.** Asking Amazon to
30
- BUILD a report is a POST, and standing rule 1 names the Reports API as the required bulk path, so
31
- refusing it would leave only a quiet page loop over entity endpoints -- the thing that rule exists to
32
- forbid. It is shaped like `modules/collections_send.py`'s `mail.mail create`, the codebase's single
33
- standing precedent: **one operation, its path a module constant rather than a parameter**, so no
34
- caller can redirect it. A report request creates a JOB, not seller data: it cannot alter a listing,
35
- a price, an order or an inventory level, and it cannot send a buyer anything. Full reasoning sits on
36
- `_REPORTS_CREATE_PATH`.
37
 
38
  β›” **THE BULK PATH IS THE REPORTS API, NEVER PAGE-BY-PAGE.** SP-API is token-bucket rate limited per
39
  operation per selling-partner/application pair, and Amazon treats throttling as evidence of a wrong
@@ -75,6 +73,18 @@ REPORT_BURST = 15
75
  #: https://developer-docs.amazon.com/sp-api/reference/createreport
76
  REPORT_SUSTAINED_SECONDS = 60.0
77
 
 
 
 
 
 
 
 
 
 
 
 
 
78
  # ⭐ One process-wide bucket and lock prevent concurrent syncs from spending the same token.
79
  # The contextual wait belongs to the current report_document logger, never another sync's log.
80
  _report_pacer = {
@@ -85,6 +95,13 @@ _report_pacer = {
85
  "waited": contextvars.ContextVar("amazon_report_create_waited", default=0.0),
86
  }
87
 
 
 
 
 
 
 
 
88
  #: ⚠ MUST MATCH `core.keychain.ENTRY_TYPES` OR `add_entry` 400s with a message about our data
89
  #: structure on a screen about their Amazon account. That was the wave-31 Meta defect and the
90
  #: comments in `keychain.py` warn about it in as many words.
@@ -120,6 +137,14 @@ SANDBOX_ENDPOINTS = {
120
  }
121
  DEFAULT_REGION = "na"
122
 
 
 
 
 
 
 
 
 
123
 
124
  def is_sandbox(fields):
125
  """Does this stored credential belong to a SANDBOX app? -> bool.
@@ -227,7 +252,8 @@ REPORT_TYPES = {
227
  },
228
  "inventory": {
229
  "report": "GET_LEDGER_SUMMARY_VIEW_DATA",
230
- "validates_against": "inventory ledger summary vs ledger detail",
 
231
  "role": "Amazon Fulfillment / Amazon Warehousing and Distribution",
232
  },
233
  # β›”β›” O5 IS CLOSED AND THE ANSWER IS NO (researched 2026-09-04). All 101 report types were
@@ -246,8 +272,8 @@ REPORT_TYPES = {
246
  "pricing": {
247
  "report": "GET_SALES_AND_TRAFFIC_REPORT",
248
  "report_covers": "buy box SHARE only; PRICE has no bulk path",
249
- "price_path": "productPricing/2022-05-01 getCompetitiveSummary (batch 20, 0.033/sec)",
250
- "validates_against": "sales and traffic report totals vs our rows",
251
  "role": "Pricing + Brand Analytics",
252
  },
253
  "brand_analytics": {
@@ -305,7 +331,7 @@ def _safe(url):
305
  return "<url>"
306
 
307
 
308
- def _retry(attempt_fn, what, *, sleep=time.sleep, rand=random.random):
309
  """Retry only SP-API throttling; return successes and propagate other refusals unchanged."""
310
  for n in range(1, RETRY_MAX_ATTEMPTS + 1):
311
  try:
@@ -315,10 +341,11 @@ def _retry(attempt_fn, what, *, sleep=time.sleep, rand=random.random):
315
  # β›” probe_credential exposes str(e)[:200]; amazon_store.sync stores str(e)[:160].
316
  # Throttle, cause AND the complete fix must fit in the first 160 characters;
317
  # this head is 152 with the named cap. Operation/URL detail must stay AFTER it.
 
318
  raise AmazonThrottled(
319
  f"Amazon throttled (429); {n - 1} retries spent. "
320
  "Cause: SP-API limits per operation, app and seller. "
321
- "Fix: sync fewer tables at once and widen the report window. "
322
  f"Operation: {_safe(what)}", retry_after=e.retry_after) from None
323
  wait = min(RETRY_MAX_SECONDS, RETRY_BASE_SECONDS * 2 ** (n - 1))
324
  floor = min(RETRY_MAX_SECONDS, e.retry_after or 0.0)
@@ -348,6 +375,19 @@ def _pace_report_create(*, sleep=time.sleep, now=time.monotonic):
348
  return wait
349
 
350
 
 
 
 
 
 
 
 
 
 
 
 
 
 
351
  def _post_form(url, data, timeout=_TIMEOUT):
352
  body = urllib.parse.urlencode(data).encode("ascii")
353
  req = urllib.request.Request(url, data=body, method="POST")
@@ -393,7 +433,7 @@ def access_token(fields, timeout=_TIMEOUT):
393
 
394
 
395
  def _request(path, token, region=DEFAULT_REGION, method="GET", timeout=_TIMEOUT,
396
- sandbox=False, **params):
397
  """One read-only SP-API call. Returns the decoded JSON body.
398
 
399
  β›” NON-GET IS REFUSED HERE, BEFORE THE REQUEST IS BUILT. There is no flag, no keyword and no
@@ -435,9 +475,10 @@ def _request(path, token, region=DEFAULT_REGION, method="GET", timeout=_TIMEOUT,
435
  detail = ""
436
  # 429 is the rate limiter, and standing rule 1 wants it NAMED rather than swallowed.
437
  if e.code == 429:
 
438
  raise AmazonThrottled(
439
  f"{_safe(url)} was throttled (429). SP-API is token-bucket limited per operation; "
440
- "the fix is the Reports API bulk path, not a slower page loop",
441
  retry_after=(e.headers or {}).get("Retry-After")) from None
442
  raise AmazonError(f"{_safe(url)} answered {e.code}"
443
  + (f": {detail}" if detail else "")) from None
@@ -446,10 +487,161 @@ def _request(path, token, region=DEFAULT_REGION, method="GET", timeout=_TIMEOUT,
446
 
447
  base = (SANDBOX_ENDPOINTS if sandbox else ENDPOINTS).get(
448
  str(region or DEFAULT_REGION).lower(), "")
449
- return _retry(attempt, f"{base}/{str(path).lstrip('/')}")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
450
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
451
 
452
- #: β›”β›” THE ONE NON-GET PATH IN THIS MODULE, AND IT IS A CONSTANT RATHER THAN A PARAMETER.
 
453
  #:
454
  #: The Reports API bulk path cannot be reached with GET alone: asking Amazon to BUILD a report is
455
  #: `POST /reports/2021-06-30/reports`. Standing rule 1 forbids a silent cap on a connector's
@@ -463,8 +655,9 @@ def _request(path, token, region=DEFAULT_REGION, method="GET", timeout=_TIMEOUT,
463
  #: caller-supplied value to smuggle another operation through.
464
  #:
465
  #: ⚠ AND THE BLAST RADIUS IS WORTH STATING PLAINLY, because "read-only" is a promise to the owner
466
- #: and this is the one place it bends: a report request creates a JOB, not seller data. It cannot
467
- #: alter a listing, a price, an order or an inventory level, and it cannot send a buyer anything.
 
468
  #: The two roles that could do those things were REFUSED at registration (R7).
469
  _REPORTS_CREATE_PATH = "reports/2021-06-30/reports"
470
 
@@ -529,7 +722,9 @@ def _create_report(token, report_type, marketplace_ids, region=DEFAULT_REGION,
529
  str(region or DEFAULT_REGION).lower(), "")
530
  # ⭐ Pace once per new report. Retrying that same report pays only the shared backoff.
531
  _report_pacer["waited"].set(_pace_report_create())
532
- return _retry(attempt, f"{base}/{_REPORTS_CREATE_PATH}")
 
 
533
 
534
 
535
  #: Terminal processing states. `DONE` is the only one that yields a document.
@@ -640,5 +835,3 @@ def marketplace_participations(fields, timeout=_TIMEOUT):
640
  got = _request("sellers/v1/marketplaceParticipations", token, region=region, timeout=timeout,
641
  sandbox=is_sandbox(fields))
642
  return got.get("payload") or []
643
-
644
-
 
21
  signing requests is describing a dead API.
22
  Β· https://developer-docs.amazon.com/sp-api/changelog/sp-api-will-no-longer-require-aws-iam-or-aws-signature-version-4
23
 
24
+ β›” **READ-ONLY, ENFORCED HERE, WITH EXACTLY TWO NAMED EXCEPTIONS.** `_request` refuses any method but
25
  GET before it builds a request. SP-API can create feeds, send buyer messages and adjust listings;
26
  none of that is reachable from this module, and the refusal sits below every route and payload so no
27
  caller can opt out.
28
 
29
+ ⚠ **The exceptions are `_create_report` and `_competitive_summary_batch`, and both are declared
30
+ rather than hidden.** Asking Amazon to BUILD a report is a POST, and Product Pricing's
31
+ `getCompetitiveSummary` is a POST-wrapped batch of read-only GET subrequests. Both functions have
32
+ their path as a module constant rather than a parameter, so no caller can redirect either one. They
33
+ cannot alter a listing, price, order or inventory level, and cannot send a buyer anything. Full
34
+ reasoning sits on their path constants.
 
 
35
 
36
  β›” **THE BULK PATH IS THE REPORTS API, NEVER PAGE-BY-PAGE.** SP-API is token-bucket rate limited per
37
  operation per selling-partner/application pair, and Amazon treats throttling as evidence of a wrong
 
73
  #: https://developer-docs.amazon.com/sp-api/reference/createreport
74
  REPORT_SUSTAINED_SECONDS = 60.0
75
 
76
+ #: Product Pricing v2022-05-01 `getCompetitiveSummary`, official default usage plan.
77
+ #: https://developer-docs.amazon.com/sp-api/reference/getcompetitivesummary
78
+ COMPETITIVE_SUMMARY_RATE = 0.033
79
+ COMPETITIVE_SUMMARY_INTERVAL = 1.0 / COMPETITIVE_SUMMARY_RATE
80
+ COMPETITIVE_SUMMARY_BATCH_SIZE = 20
81
+
82
+ #: AWD v2024-05-09 `listInventory`, official default usage plan and maximum page size.
83
+ #: https://developer-docs.amazon.com/sp-api/reference/listinventory
84
+ AWD_INVENTORY_RATE = 2.0
85
+ AWD_INVENTORY_INTERVAL = 1.0 / AWD_INVENTORY_RATE
86
+ AWD_INVENTORY_PAGE_SIZE = 200
87
+
88
  # ⭐ One process-wide bucket and lock prevent concurrent syncs from spending the same token.
89
  # The contextual wait belongs to the current report_document logger, never another sync's log.
90
  _report_pacer = {
 
95
  "waited": contextvars.ContextVar("amazon_report_create_waited", default=0.0),
96
  }
97
 
98
+ #: One process-wide reservation per entity operation. Holding the lock through the short wait keeps
99
+ #: concurrent tenant syncs from spending the same operation token.
100
+ _entity_pacers = {
101
+ "competitive_summary": {"last": None, "lock": threading.Lock()},
102
+ "awd_inventory": {"last": None, "lock": threading.Lock()},
103
+ }
104
+
105
  #: ⚠ MUST MATCH `core.keychain.ENTRY_TYPES` OR `add_entry` 400s with a message about our data
106
  #: structure on a screen about their Amazon account. That was the wave-31 Meta defect and the
107
  #: comments in `keychain.py` warn about it in as many words.
 
137
  }
138
  DEFAULT_REGION = "na"
139
 
140
+ #: These paths are constants, not caller input. The outer Product Pricing request is POST, but
141
+ #: every request inside its batch is a fixed GET to the read-only item URI.
142
+ _COMPETITIVE_SUMMARY_BATCH_PATH = (
143
+ "batches/products/pricing/2022-05-01/items/competitiveSummary")
144
+ _COMPETITIVE_SUMMARY_ITEM_URI = "/products/pricing/2022-05-01/items/competitiveSummary"
145
+ _COMPETITIVE_SUMMARY_INCLUDED = ("featuredBuyingOptions", "referencePrices")
146
+ _AWD_INVENTORY_PATH = "awd/2024-05-09/inventory"
147
+
148
 
149
  def is_sandbox(fields):
150
  """Does this stored credential belong to a SANDBOX app? -> bool.
 
252
  },
253
  "inventory": {
254
  "report": "GET_LEDGER_SUMMARY_VIEW_DATA",
255
+ "entity_read": "AWD v2024-05-09 listInventory (all 200-row pages)",
256
+ "validates_against": "FBA ledger summary vs detail and AWD API items vs stored AWD rows",
257
  "role": "Amazon Fulfillment / Amazon Warehousing and Distribution",
258
  },
259
  # β›”β›” O5 IS CLOSED AND THE ANSWER IS NO (researched 2026-09-04). All 101 report types were
 
272
  "pricing": {
273
  "report": "GET_SALES_AND_TRAFFIC_REPORT",
274
  "report_covers": "buy box SHARE only; PRICE has no bulk path",
275
+ "price_path": "Product Pricing v2022-05-01 getCompetitiveSummary (batch 20, 0.033/sec)",
276
+ "validates_against": "sales and traffic report totals and competitive ASIN responses",
277
  "role": "Pricing + Brand Analytics",
278
  },
279
  "brand_analytics": {
 
331
  return "<url>"
332
 
333
 
334
+ def _retry(attempt_fn, what, *, sleep=time.sleep, rand=random.random, exhausted_fix=None):
335
  """Retry only SP-API throttling; return successes and propagate other refusals unchanged."""
336
  for n in range(1, RETRY_MAX_ATTEMPTS + 1):
337
  try:
 
341
  # β›” probe_credential exposes str(e)[:200]; amazon_store.sync stores str(e)[:160].
342
  # Throttle, cause AND the complete fix must fit in the first 160 characters;
343
  # this head is 152 with the named cap. Operation/URL detail must stay AFTER it.
344
+ fix = exhausted_fix or "sync fewer tables at once and widen the report window"
345
  raise AmazonThrottled(
346
  f"Amazon throttled (429); {n - 1} retries spent. "
347
  "Cause: SP-API limits per operation, app and seller. "
348
+ f"Fix: {fix}. "
349
  f"Operation: {_safe(what)}", retry_after=e.retry_after) from None
350
  wait = min(RETRY_MAX_SECONDS, RETRY_BASE_SECONDS * 2 ** (n - 1))
351
  floor = min(RETRY_MAX_SECONDS, e.retry_after or 0.0)
 
375
  return wait
376
 
377
 
378
+ def _pace_entity(name, interval, *, sleep=time.sleep, now=time.monotonic):
379
+ """Reserve one call for a fixed entity operation under Amazon's default usage plan."""
380
+ pacer = _entity_pacers[name]
381
+ with pacer["lock"]:
382
+ stamp = now()
383
+ last = pacer["last"]
384
+ wait = max(0.0, last + interval - stamp) if last is not None else 0.0
385
+ if wait:
386
+ sleep(wait)
387
+ pacer["last"] = max(stamp + wait, now())
388
+ return wait
389
+
390
+
391
  def _post_form(url, data, timeout=_TIMEOUT):
392
  body = urllib.parse.urlencode(data).encode("ascii")
393
  req = urllib.request.Request(url, data=body, method="POST")
 
433
 
434
 
435
  def _request(path, token, region=DEFAULT_REGION, method="GET", timeout=_TIMEOUT,
436
+ sandbox=False, throttle_fix=None, **params):
437
  """One read-only SP-API call. Returns the decoded JSON body.
438
 
439
  β›” NON-GET IS REFUSED HERE, BEFORE THE REQUEST IS BUILT. There is no flag, no keyword and no
 
475
  detail = ""
476
  # 429 is the rate limiter, and standing rule 1 wants it NAMED rather than swallowed.
477
  if e.code == 429:
478
+ fix = throttle_fix or "use the Reports API bulk path instead of a slower page loop"
479
  raise AmazonThrottled(
480
  f"{_safe(url)} was throttled (429). SP-API is token-bucket limited per operation; "
481
+ f"the fix is to {fix}",
482
  retry_after=(e.headers or {}).get("Retry-After")) from None
483
  raise AmazonError(f"{_safe(url)} answered {e.code}"
484
  + (f": {detail}" if detail else "")) from None
 
487
 
488
  base = (SANDBOX_ENDPOINTS if sandbox else ENDPOINTS).get(
489
  str(region or DEFAULT_REGION).lower(), "")
490
+ return _retry(
491
+ attempt, f"{base}/{str(path).lstrip('/')}",
492
+ exhausted_fix=(throttle_fix
493
+ or "use the Reports API bulk path instead of a slower page loop"))
494
+
495
+
496
+ #: β›” THE SECOND NON-GET SP-API PATH. `getCompetitiveSummary` is an Amazon-defined POST wrapper,
497
+ #: but every item in its body is a GET to `_COMPETITIVE_SUMMARY_ITEM_URI`. The path, item method,
498
+ #: item URI, included data and 20-item maximum are all fixed below; no caller can turn this into a
499
+ #: write-shaped Product Pricing request.
500
+ def _competitive_summary_batch(token, asins, marketplace_id, region=DEFAULT_REGION,
501
+ timeout=_TIMEOUT, sandbox=False):
502
+ """Read one fixed `getCompetitiveSummary` batch. -> its response list."""
503
+ items = [str(asin or "").strip() for asin in (asins or ()) if str(asin or "").strip()]
504
+ if not 1 <= len(items) <= COMPETITIVE_SUMMARY_BATCH_SIZE:
505
+ raise AmazonError(
506
+ f"competitive summary needs 1 to {COMPETITIVE_SUMMARY_BATCH_SIZE} ASINs per batch")
507
+ marketplace = str(marketplace_id or "").strip()
508
+ if not marketplace:
509
+ raise AmazonError("competitive summary needs a marketplace_id")
510
+
511
+ body = {"requests": [{
512
+ "asin": asin,
513
+ "marketplaceId": marketplace,
514
+ "includedData": list(_COMPETITIVE_SUMMARY_INCLUDED),
515
+ "method": "GET",
516
+ "uri": _COMPETITIVE_SUMMARY_ITEM_URI,
517
+ } for asin in items]}
518
+
519
+ def attempt():
520
+ base = (SANDBOX_ENDPOINTS if sandbox else ENDPOINTS).get(
521
+ str(region or DEFAULT_REGION).lower())
522
+ if not base:
523
+ raise AmazonError(f"unknown region `{region}`")
524
+ url = f"{base}/{_COMPETITIVE_SUMMARY_BATCH_PATH}"
525
+ req = urllib.request.Request(url, data=json.dumps(body).encode("utf-8"), method="POST")
526
+ req.add_header("x-amz-access-token", token)
527
+ req.add_header("Content-Type", "application/json")
528
+ req.add_header("Accept", "application/json")
529
+ req.add_header("User-Agent", _UA)
530
+ try:
531
+ with urllib.request.urlopen(req, timeout=timeout) as r:
532
+ got = json.loads(r.read().decode("utf-8") or "{}")
533
+ except urllib.error.HTTPError as e:
534
+ detail = ""
535
+ try:
536
+ errs = (json.loads((e.read() or b"").decode("utf-8") or "{}").get("errors") or [])
537
+ detail = str((errs[0] or {}).get("message") or "")[:200] if errs else ""
538
+ except Exception: # noqa: BLE001
539
+ detail = ""
540
+ if e.code == 429:
541
+ raise AmazonThrottled(
542
+ f"{_safe(url)} was throttled (429). getCompetitiveSummary defaults to "
543
+ "0.033 requests/second with a burst of 1; keep 20 ASINs per batch",
544
+ retry_after=(e.headers or {}).get("Retry-After")) from None
545
+ if e.code == 403:
546
+ raise AmazonError(
547
+ f"{_safe(url)} refused getCompetitiveSummary (403). The application needs "
548
+ "the Pricing role") from None
549
+ raise AmazonError(f"{_safe(url)} answered {e.code}"
550
+ + (f": {detail}" if detail else "")) from None
551
+ except Exception as e: # noqa: BLE001
552
+ raise AmazonError(f"{_safe(url)} unreachable: {type(e).__name__}") from None
553
+ responses = got.get("responses")
554
+ if not isinstance(responses, list):
555
+ raise AmazonError("getCompetitiveSummary returned no response list")
556
+ return responses
557
+
558
+ base = (SANDBOX_ENDPOINTS if sandbox else ENDPOINTS).get(
559
+ str(region or DEFAULT_REGION).lower(), "")
560
+ return _retry(
561
+ attempt, f"{base}/{_COMPETITIVE_SUMMARY_BATCH_PATH}",
562
+ exhausted_fix="retain 20 ASINs per batch and lower the batch request rate")
563
+
564
+
565
+ def competitive_summaries(fields, asins, timeout=_TIMEOUT, *, sleep=time.sleep,
566
+ now=time.monotonic):
567
+ """Read competitive summaries for every distinct ASIN, in complete fixed-size batches."""
568
+ unique = []
569
+ seen = set()
570
+ for raw in asins or ():
571
+ asin = str(raw or "").strip()
572
+ if asin and asin not in seen:
573
+ unique.append(asin)
574
+ seen.add(asin)
575
+ if not unique:
576
+ return []
577
 
578
+ f = fields or {}
579
+ region = str(f.get("region") or DEFAULT_REGION)
580
+ marketplace = str(f.get("marketplace_id") or US_MARKETPLACE_ID)
581
+ sandbox = is_sandbox(f)
582
+ token = access_token(f, timeout=timeout)
583
+ token_issued = now()
584
+ out = []
585
+ for offset in range(0, len(unique), COMPETITIVE_SUMMARY_BATCH_SIZE):
586
+ _pace_entity("competitive_summary", COMPETITIVE_SUMMARY_INTERVAL,
587
+ sleep=sleep, now=now)
588
+ if now() - token_issued >= 3000:
589
+ token = access_token(f, timeout=timeout)
590
+ token_issued = now()
591
+ batch = unique[offset:offset + COMPETITIVE_SUMMARY_BATCH_SIZE]
592
+ responses = _competitive_summary_batch(
593
+ token, batch, marketplace, region=region, timeout=timeout, sandbox=sandbox)
594
+ returned = [str(((r or {}).get("body") or {}).get("asin") or "") for r in responses]
595
+ if len(responses) != len(batch) or sorted(returned) != sorted(batch):
596
+ raise AmazonError(
597
+ "getCompetitiveSummary returned an incomplete or mismatched ASIN batch; refusing "
598
+ "to treat a partial response as complete")
599
+ out.extend(responses)
600
+ return out
601
+
602
+
603
+ def awd_inventory(fields, timeout=_TIMEOUT, *, sleep=time.sleep, now=time.monotonic):
604
+ """Read every AWD inventory page, including through empty pages. -> inventory summaries."""
605
+ f = fields or {}
606
+ region = str(f.get("region") or DEFAULT_REGION)
607
+ sandbox = is_sandbox(f)
608
+ token = access_token(f, timeout=timeout)
609
+ token_issued = now()
610
+ next_token = None
611
+ seen_tokens = set()
612
+ rows = []
613
+ while True:
614
+ _pace_entity("awd_inventory", AWD_INVENTORY_INTERVAL, sleep=sleep, now=now)
615
+ if now() - token_issued >= 3000:
616
+ token = access_token(f, timeout=timeout)
617
+ token_issued = now()
618
+ try:
619
+ got = _request(
620
+ _AWD_INVENTORY_PATH, token, region=region, timeout=timeout, sandbox=sandbox,
621
+ throttle_fix="resume the same listInventory pagination later without dropping pages",
622
+ details="SHOW", maxResults=AWD_INVENTORY_PAGE_SIZE, nextToken=next_token)
623
+ except AmazonError as e:
624
+ if "answered 403" in str(e):
625
+ raise AmazonError(
626
+ f"{_AWD_INVENTORY_PATH} was refused (403). The application needs the Amazon "
627
+ "Warehousing and Distribution role") from None
628
+ raise
629
+ page = got.get("inventory")
630
+ if not isinstance(page, list):
631
+ raise AmazonError("listInventory returned no inventory list")
632
+ rows.extend(page)
633
+ following = str(got.get("nextToken") or "").strip()
634
+ if not following:
635
+ break
636
+ if following in seen_tokens:
637
+ raise AmazonError(
638
+ "listInventory repeated a nextToken; refusing an infinite or duplicate page loop")
639
+ seen_tokens.add(following)
640
+ next_token = following
641
+ return rows
642
 
643
+
644
+ #: β›”β›” THE REPORT-JOB NON-GET PATH, AND IT IS A CONSTANT RATHER THAN A PARAMETER.
645
  #:
646
  #: The Reports API bulk path cannot be reached with GET alone: asking Amazon to BUILD a report is
647
  #: `POST /reports/2021-06-30/reports`. Standing rule 1 forbids a silent cap on a connector's
 
655
  #: caller-supplied value to smuggle another operation through.
656
  #:
657
  #: ⚠ AND THE BLAST RADIUS IS WORTH STATING PLAINLY, because "read-only" is a promise to the owner
658
+ #: and this is one of two places the HTTP verb bends: a report request creates a JOB, not seller
659
+ #: data. The other is Amazon's POST-wrapped batch of fixed competitive-summary GETs. Neither can
660
+ #: alter a listing, a price, an order or an inventory level, and neither can send a buyer anything.
661
  #: The two roles that could do those things were REFUSED at registration (R7).
662
  _REPORTS_CREATE_PATH = "reports/2021-06-30/reports"
663
 
 
722
  str(region or DEFAULT_REGION).lower(), "")
723
  # ⭐ Pace once per new report. Retrying that same report pays only the shared backoff.
724
  _report_pacer["waited"].set(_pace_report_create())
725
+ return _retry(
726
+ attempt, f"{base}/{_REPORTS_CREATE_PATH}",
727
+ exhausted_fix="create fewer reports with wider report windows")
728
 
729
 
730
  #: Terminal processing states. `DONE` is the only one that yields a document.
 
835
  got = _request("sellers/v1/marketplaceParticipations", token, region=region, timeout=timeout,
836
  sandbox=is_sandbox(fields))
837
  return got.get("payload") or []
 
 
platform/harness/amazon_store.py CHANGED
@@ -1,9 +1,10 @@
1
  """amazon_store.py β€” the Reports-API loader for the tenant mirror, and the only place Amazon rows
2
- are written.
3
 
4
  python platform/harness/amazon_store.py --sync pull into tenant #0's mirror
5
  python platform/harness/amazon_store.py --sync --tenant gtmlab
6
  python platform/harness/amazon_store.py --status what is in the mirror now
 
7
  python platform/harness/amazon_store.py --selftest parsers, typing, naming, validate
8
 
9
  ⭐ THE SHAPE, and it is deliberately the third instance of one pattern rather than a new one:
@@ -65,6 +66,14 @@ PREFIX = "amzn_"
65
  TABLES = ("orders", "order_items", "listings", "settlements", "inventory",
66
  "pricing", "brand_analytics")
67
 
 
 
 
 
 
 
 
 
68
  #: Mirror table names. The `ut_amzn_*` grids are spawned from these by `amazon_relational`.
69
  def table_name(table):
70
  """`orders` -> `amzn_orders`. One place, so a rename cannot half-land."""
@@ -204,6 +213,70 @@ def _flatten_row(obj, prefix=""):
204
  return out
205
 
206
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
207
  def merge_columns(pinned, parsed):
208
  """The column set for a `schema_artifact` table: the pin, plus anything the payload really had.
209
 
@@ -387,6 +460,188 @@ def credential(tenant_key="royal-imports", rt=None):
387
  return None
388
 
389
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
390
  def _client():
391
  """The SP-API transport, from the DATA LAYER.
392
 
@@ -404,6 +659,28 @@ def _client():
404
  return amazon_client
405
 
406
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
407
  def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None,
408
  max_wait=900):
409
  """Pull every declared table into the tenant's mirror. -> a report dict.
@@ -440,7 +717,7 @@ def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None
440
  want = [t for t in (tables or TABLES) if t in TABLES]
441
  end = datetime.now(timezone.utc)
442
  start = since or (end - timedelta(days=DEFAULT_BACKFILL_DAYS))
443
- con = datastore.connect(datastore.path_for(rt) if rt is not None else None)
444
  report = {"tenant": tenant_key, "tables": {}, "started": _iso(end)}
445
 
446
  for table in want:
@@ -454,7 +731,8 @@ def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None
454
  prov = amazon_schema.PROVENANCE.get(table, "measured_at_sync")
455
  root = (amazon_schema.ARTIFACTS.get(table) or (None, None))[1]
456
  try:
457
- written, cols = 0, []
 
458
  for w0, w1 in _windows(start, end):
459
  text, meta = cx.report_document(
460
  fields, rtype, start=_iso(w0), end=_iso(w1), max_wait=max_wait, log=log)
@@ -493,15 +771,78 @@ def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None
493
  continue
494
  if table == "orders":
495
  rows = _dedupe_orders(rows)
 
 
 
 
 
 
496
  types = _ensure(con, mirror, cols, rows, declared=declared)
497
- con.execute(f"DELETE FROM {mirror} WHERE 1=1" if len(_windows(start, end)) == 1
498
- else f"DELETE FROM {mirror} WHERE 1=0")
 
 
 
499
  written += _replace(con, mirror, cols, types, rows)
500
- report["tables"][table] = {"rows": written, "columns": len(cols),
501
- "provenance": prov, "report": rtype}
502
- con.execute("INSERT OR REPLACE INTO _sync_state VALUES (?,?,?,?,?,?)",
503
- [key, "done", None, written, _iso(datetime.now(timezone.utc)), None])
504
- log(f" {table:<16} {written:>7,} rows {len(cols):>3} cols {prov}")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
505
  except Exception as e: # noqa: BLE001
506
  # ⚠ ONE TABLE'S REFUSAL IS NOT THE SYNC'S. A 403 on brand_analytics is R5's UNPROVEN
507
  # ticket answering itself, and it must not cost the six tables that did work.
@@ -548,7 +889,7 @@ def validate(tenant_key="royal-imports", rt=None, log=print):
548
  [[a-check-that-cannot-reach-green-is-broken]].
549
  """
550
  rt = rt if rt is not None else _runtime_for(tenant_key)
551
- con = datastore.connect(datastore.path_for(rt) if rt is not None else None)
552
  cx = _client()
553
  out = {}
554
  for table in TABLES:
@@ -563,6 +904,29 @@ def validate(tenant_key="royal-imports", rt=None, log=print):
563
  out[table] = {"state": "unproven", "reason": "table is empty",
564
  "reconciles": spec.get("validates_against")}
565
  continue
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
566
  out[table] = {"state": "ok", "rows": n, "reconciles": spec.get("validates_against")}
567
  for t, r in out.items():
568
  log(f" {t:<16} {r['state']:<10} {r.get('rows', 0):>7,} {r.get('reconciles') or ''}")
@@ -573,7 +937,7 @@ def status(tenant_key="royal-imports", rt=None):
573
  """What is in the mirror now. Needs no credential and no network."""
574
  rt = rt if rt is not None else _runtime_for(tenant_key)
575
  try:
576
- con = datastore.connect(datastore.path_for(rt) if rt is not None else None)
577
  except Exception as e: # noqa: BLE001
578
  return {"error": f"{type(e).__name__}: {e}"}
579
  out = {}
@@ -586,6 +950,8 @@ def status(tenant_key="royal-imports", rt=None):
586
  n, cols = None, 0
587
  out[table] = {"rows": n, "columns": cols,
588
  "provenance": amazon_schema.PROVENANCE.get(table)}
 
 
589
  return out
590
 
591
 
@@ -710,7 +1076,150 @@ def selftest(log=print):
710
  ck("NC-AZ15 an empty document yields no columns and no rows",
711
  parse_flat_file("") == ([], []) and parse_json_report("", "x") == ([], []))
712
 
713
- log(f"\n{'GREEN' if not fails else 'RED'}: {15 - len(fails)}/15")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
714
  return 0 if not fails else 1
715
 
716
 
@@ -719,21 +1228,34 @@ def main(argv=None):
719
  ap.add_argument("--sync", action="store_true")
720
  ap.add_argument("--status", action="store_true")
721
  ap.add_argument("--validate", action="store_true")
 
 
722
  ap.add_argument("--selftest", action="store_true")
723
- ap.add_argument("--tenant", default="royal-imports")
 
 
724
  ap.add_argument("--table", action="append", help="limit to one table (repeatable)")
725
  a = ap.parse_args(argv)
 
 
 
 
 
 
 
 
 
726
  if a.selftest:
727
  return selftest()
728
  if a.status:
729
- print(json.dumps(status(a.tenant), indent=2))
730
  return 0
731
  if a.validate:
732
- validate(a.tenant)
733
  return 0
734
  if a.sync:
735
  t0 = time.time()
736
- rep = sync(a.tenant, tables=a.table)
737
  print(json.dumps(rep, indent=2))
738
  print(f"\n{time.time() - t0:.1f}s")
739
  return 0
 
1
  """amazon_store.py β€” the Reports-API loader for the tenant mirror, and the only place Amazon rows
2
+ are written. Reports remain the bulk spine; Pricing and AWD use their exact read-only operations.
3
 
4
  python platform/harness/amazon_store.py --sync pull into tenant #0's mirror
5
  python platform/harness/amazon_store.py --sync --tenant gtmlab
6
  python platform/harness/amazon_store.py --status what is in the mirror now
7
+ python platform/harness/amazon_store.py --purge --tenant gtmlab --confirm-tenant gtmlab
8
  python platform/harness/amazon_store.py --selftest parsers, typing, naming, validate
9
 
10
  ⭐ THE SHAPE, and it is deliberately the third instance of one pattern rather than a new one:
 
66
  TABLES = ("orders", "order_items", "listings", "settlements", "inventory",
67
  "pricing", "brand_analytics")
68
 
69
+ #: Durable proof that the two non-report role operations actually ran. A report-backed table can
70
+ #: contain rows while one of these role grants is still missing, so table presence alone is not
71
+ #: evidence that Pricing or AWD was authorized.
72
+ ENTITY_STATE_KEYS = {
73
+ "pricing": "amzn_pricing_competitive_summary",
74
+ "inventory": "amzn_inventory_awd",
75
+ }
76
+
77
  #: Mirror table names. The `ut_amzn_*` grids are spawned from these by `amazon_relational`.
78
  def table_name(table):
79
  """`orders` -> `amzn_orders`. One place, so a rename cannot half-land."""
 
213
  return out
214
 
215
 
216
+ def _competitive_rows(responses):
217
+ """Amazon batch responses -> complete flattened enrichments keyed by returned ASIN."""
218
+ out = []
219
+ for response in responses or ():
220
+ body = (response or {}).get("body") or {}
221
+ asin = str(body.get("asin") or "").strip()
222
+ if not asin:
223
+ continue
224
+ row = _flatten_row(response or {}, "competitive")
225
+ row["competitive_body_asin"] = asin
226
+ out.append(row)
227
+ return out
228
+
229
+
230
+ def _awd_rows(inventory):
231
+ """AWD inventory summaries -> flattened rows that remain distinguishable from FBA ledger."""
232
+ rows = []
233
+ for item in inventory or ():
234
+ row = _flatten_row(item or {})
235
+ row["inventory_source"] = "AWD"
236
+ rows.append(row)
237
+ return rows
238
+
239
+
240
+ def _tag_fba_inventory(rows):
241
+ """Mark report rows so one combined inventory table never conflates FBA with AWD."""
242
+ for row in rows or ():
243
+ row["inventory_source"] = "FBA"
244
+ return rows
245
+
246
+
247
+ def _pricing_asins(rows):
248
+ """Distinct child ASINs from Sales and Traffic rows, preserving source order."""
249
+ seen, out = set(), []
250
+ for row in rows or ():
251
+ asin = str(row.get("child_asin") or row.get("parent_asin") or "").strip()
252
+ if asin and asin not in seen:
253
+ seen.add(asin)
254
+ out.append(asin)
255
+ return out
256
+
257
+
258
+ def _apply_competitive(con, table, responses):
259
+ """Add every competitive response field to all matching Sales and Traffic ASIN rows."""
260
+ rows = _competitive_rows(responses)
261
+ if not rows:
262
+ return 0, 0
263
+ cols = sorted({key for row in rows for key in row})
264
+ types = _ensure(con, table, cols, rows)
265
+ assignments = ", ".join(f'"{c}"=?' for c in cols)
266
+ updated = 0
267
+ for row in rows:
268
+ asin = row["competitive_body_asin"]
269
+ values = [_cell(row.get(c), types.get(c, "VARCHAR")) for c in cols]
270
+ result = con.execute(
271
+ f'UPDATE {table} SET {assignments} WHERE child_asin=? OR parent_asin=?',
272
+ values + [asin, asin])
273
+ try:
274
+ updated += int(result.fetchone()[0])
275
+ except Exception: # noqa: BLE001
276
+ pass
277
+ return len(rows), updated
278
+
279
+
280
  def merge_columns(pinned, parsed):
281
  """The column set for a `schema_artifact` table: the pin, plus anything the payload really had.
282
 
 
460
  return None
461
 
462
 
463
+ def _tenant_key(value):
464
+ """A non-empty canonical tenant slug for a destructive or store-binding operation."""
465
+ key = str(value or "").strip().lower()
466
+ if not key:
467
+ raise ValueError("an explicit tenant slug is required")
468
+ return key
469
+
470
+
471
+ def _tenant_mirror_path(tenant_key, rt=None):
472
+ """Resolve the exact tenant mirror path without opening or creating it."""
473
+ key = _tenant_key(tenant_key)
474
+ if rt is not None:
475
+ runtime_key = str(getattr(rt, "key", "") or "").strip().lower()
476
+ if runtime_key != key:
477
+ raise RuntimeError(
478
+ f"runtime tenant {runtime_key!r} does not match requested tenant {key!r}")
479
+ return Path(rt.datastore_path)
480
+ return Path(datastore.path_for(key))
481
+
482
+
483
+ def _connect_tenant_mirror(tenant_key, rt=None):
484
+ """Bind this single-tenant process to the requested mirror, then open a writer cursor.
485
+
486
+ `datastore.connect()` deliberately takes no path argument. The safe path switch is
487
+ `datastore.use_path()`, which closes the cached writer and invalidates every read cursor before
488
+ changing the process-wide file. Calling `connect(path)` looks plausible but raises before a
489
+ credentialed Amazon sync can write its first row.
490
+ """
491
+ path = _tenant_mirror_path(tenant_key, rt)
492
+ if Path(datastore.DB_PATH).resolve() != path.resolve():
493
+ datastore.use_path(path)
494
+ if rt is not None:
495
+ rt.assert_datastore_matches()
496
+ return datastore.connect(), path
497
+
498
+
499
+ def _require_purge_confirmation(tenant_key, confirmation):
500
+ """Require the operator to repeat the exact canonical tenant slug."""
501
+ key = _tenant_key(tenant_key)
502
+ if str(confirmation or "").strip() != key:
503
+ raise RuntimeError(
504
+ f"refusing permanent deletion: --confirm-tenant must be exactly {key!r}")
505
+ return key
506
+
507
+
508
+ def _assert_amazon_credential_revoked(rt):
509
+ """Refuse mirror deletion while any Amazon credential metadata still exists.
510
+
511
+ This checks metadata, not decrypted fields. It therefore still catches a credential when the
512
+ keychain encryption key is unavailable, and an unreadable tenant store raises rather than
513
+ masquerading as an empty keychain.
514
+ """
515
+ from core import keychain as kc
516
+ if kc.first_entry_of_type(rt, "amazon_sp") is not None:
517
+ raise RuntimeError(
518
+ "refusing permanent deletion: this tenant still has a stored Amazon credential")
519
+ return True
520
+
521
+
522
+ def _quoted_identifier(value):
523
+ """Quote one database-discovered identifier for a DROP statement."""
524
+ return '"' + str(value).replace('"', '""') + '"'
525
+
526
+
527
+ def _amazon_objects(con):
528
+ """Every live mirror object reserved to Amazon by the `amzn_` naming contract."""
529
+ rows = con.execute(
530
+ "SELECT table_name, table_type FROM information_schema.tables "
531
+ "WHERE table_schema='main'").fetchall()
532
+ out = []
533
+ for name, kind in rows:
534
+ name = str(name)
535
+ if not name.startswith(PREFIX):
536
+ continue
537
+ out.append((name, str(kind or "BASE TABLE").upper()))
538
+ return sorted(out, key=lambda row: ("VIEW" not in row[1], row[0]))
539
+
540
+
541
+ def _sync_state_rows(con):
542
+ """The complete state table, used to prove non-Amazon markers survive a purge unchanged."""
543
+ return [tuple(row) for row in con.execute(
544
+ "SELECT entity, phase, last_id, cursor_wd, rows, updated_at "
545
+ "FROM _sync_state ORDER BY entity").fetchall()]
546
+
547
+
548
+ def _purge_mirror(con):
549
+ """Transactionally delete every `amzn_` mirror object and state marker, then read it back.
550
+
551
+ The prefix is the connector's reserved namespace, so this also catches a future Amazon table
552
+ that has not yet been added to `TABLES`. Non-Amazon tables and sync markers are snapshotted and
553
+ compared inside the transaction and again after commit. No caller supplies a table name.
554
+ """
555
+ objects = _amazon_objects(con)
556
+ before_tables = {str(row[0]) for row in con.execute(
557
+ "SELECT table_name FROM information_schema.tables "
558
+ "WHERE table_schema='main'").fetchall()}
559
+ before_other_tables = sorted(name for name in before_tables if not name.startswith(PREFIX))
560
+ before_state = _sync_state_rows(con)
561
+ amazon_state = [str(row[0]) for row in before_state if str(row[0]).startswith(PREFIX)]
562
+ other_state = [row for row in before_state if not str(row[0]).startswith(PREFIX)]
563
+ row_counts = {}
564
+ for name, _kind in objects:
565
+ row_counts[name] = int(con.execute(
566
+ f"SELECT COUNT(*) FROM {_quoted_identifier(name)}").fetchone()[0])
567
+
568
+ try:
569
+ con.execute("BEGIN TRANSACTION")
570
+ for name, kind in objects:
571
+ noun = "VIEW" if "VIEW" in kind else "TABLE"
572
+ con.execute(f"DROP {noun} {_quoted_identifier(name)}")
573
+ for entity in amazon_state:
574
+ con.execute("DELETE FROM _sync_state WHERE entity=?", [entity])
575
+
576
+ if _amazon_objects(con):
577
+ raise RuntimeError("Amazon mirror objects remain inside the purge transaction")
578
+ inside_other_tables = sorted(str(row[0]) for row in con.execute(
579
+ "SELECT table_name FROM information_schema.tables WHERE table_schema='main'").fetchall()
580
+ if not str(row[0]).startswith(PREFIX))
581
+ if inside_other_tables != before_other_tables:
582
+ raise RuntimeError("a non-Amazon mirror object changed during the purge")
583
+ inside_state = _sync_state_rows(con)
584
+ if inside_state != other_state:
585
+ raise RuntimeError("a non-Amazon sync marker changed during the purge")
586
+ con.execute("COMMIT")
587
+ except Exception:
588
+ try:
589
+ con.execute("ROLLBACK")
590
+ except Exception: # noqa: BLE001
591
+ pass
592
+ raise
593
+
594
+ residue_objects = _amazon_objects(con)
595
+ residue_state = [str(row[0]) for row in _sync_state_rows(con)
596
+ if str(row[0]).startswith(PREFIX)]
597
+ after_other_tables = sorted(str(row[0]) for row in con.execute(
598
+ "SELECT table_name FROM information_schema.tables WHERE table_schema='main'").fetchall()
599
+ if not str(row[0]).startswith(PREFIX))
600
+ after_other_state = [row for row in _sync_state_rows(con)
601
+ if not str(row[0]).startswith(PREFIX)]
602
+ if residue_objects or residue_state:
603
+ raise RuntimeError(
604
+ "Amazon mirror purge committed but independent readback found residue")
605
+ if after_other_tables != before_other_tables or after_other_state != other_state:
606
+ raise RuntimeError(
607
+ "Amazon mirror purge committed but independent readback found collateral change")
608
+ return {
609
+ "tables": [name for name, _kind in objects],
610
+ "rows": sum(row_counts.values()),
611
+ "syncMarkers": len(amazon_state),
612
+ "residueTables": [],
613
+ "residueSyncMarkers": [],
614
+ }
615
+
616
+
617
+ def purge(tenant_key, confirmation, rt=None, log=print):
618
+ """Permanently remove one tenant's live Amazon mirror after credential revocation.
619
+
620
+ This is an operator-only lifecycle control. It never revokes a credential itself and refuses
621
+ to run until the tenant runtime resolves, the exact tenant slug is repeated, the mirror already
622
+ exists, and every Amazon credential metadata row is gone.
623
+ """
624
+ key = _require_purge_confirmation(tenant_key, confirmation)
625
+ rt = rt if rt is not None else _runtime_for(key)
626
+ if rt is None:
627
+ raise RuntimeError(
628
+ f"tenant {key!r} could not be resolved; refusing to guess a database target")
629
+ path = _tenant_mirror_path(key, rt)
630
+ if not path.is_file():
631
+ raise RuntimeError(
632
+ f"tenant {key!r} has no existing mirror file; refusing to create one as purge proof")
633
+ _assert_amazon_credential_revoked(rt)
634
+ con, bound_path = _connect_tenant_mirror(key, rt)
635
+ try:
636
+ result = _purge_mirror(con)
637
+ finally:
638
+ con.close()
639
+ report = {"tenant": key, "database": bound_path.name, **result}
640
+ log(f" tenant {key}: permanently deleted {result['rows']:,} Amazon mirror rows from "
641
+ f"{len(result['tables']):,} tables and {result['syncMarkers']:,} sync markers")
642
+ return report
643
+
644
+
645
  def _client():
646
  """The SP-API transport, from the DATA LAYER.
647
 
 
659
  return amazon_client
660
 
661
 
662
+ def _record_state(con, key, phase, rows=0, detail=None):
663
+ """Write `_sync_state` in its declared column order, never by a misleading positional guess."""
664
+ con.execute(
665
+ "INSERT OR REPLACE INTO _sync_state "
666
+ "(entity, phase, last_id, cursor_wd, rows, updated_at) VALUES (?,?,?,?,?,?)",
667
+ [str(key), str(phase), None, (str(detail)[:300] if detail else None), int(rows or 0),
668
+ _iso(datetime.now(timezone.utc))])
669
+
670
+
671
+ def _read_state(con, key):
672
+ """One durable operation marker, or None when that operation has never completed."""
673
+ try:
674
+ row = con.execute(
675
+ "SELECT phase, rows, updated_at, cursor_wd FROM _sync_state WHERE entity=?",
676
+ [str(key)]).fetchone()
677
+ except Exception: # noqa: BLE001
678
+ return None
679
+ if not row:
680
+ return None
681
+ return {"state": row[0], "rows": row[1], "updated_at": row[2], "detail": row[3]}
682
+
683
+
684
  def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None,
685
  max_wait=900):
686
  """Pull every declared table into the tenant's mirror. -> a report dict.
 
717
  want = [t for t in (tables or TABLES) if t in TABLES]
718
  end = datetime.now(timezone.utc)
719
  start = since or (end - timedelta(days=DEFAULT_BACKFILL_DAYS))
720
+ con, _path = _connect_tenant_mirror(tenant_key, rt)
721
  report = {"tenant": tenant_key, "tables": {}, "started": _iso(end)}
722
 
723
  for table in want:
 
731
  prov = amazon_schema.PROVENANCE.get(table, "measured_at_sync")
732
  root = (amazon_schema.ARTIFACTS.get(table) or (None, None))[1]
733
  try:
734
+ written, cols, cleared = 0, [], False
735
+ pricing_asins = []
736
  for w0, w1 in _windows(start, end):
737
  text, meta = cx.report_document(
738
  fields, rtype, start=_iso(w0), end=_iso(w1), max_wait=max_wait, log=log)
 
771
  continue
772
  if table == "orders":
773
  rows = _dedupe_orders(rows)
774
+ if table == "inventory":
775
+ rows = _tag_fba_inventory(rows)
776
+ if "inventory_source" not in cols:
777
+ cols.append("inventory_source")
778
+ if table == "pricing":
779
+ pricing_asins.extend(_pricing_asins(rows))
780
  types = _ensure(con, mirror, cols, rows, declared=declared)
781
+ # Clear once, after the table exists. The old multi-window branch deleted nothing,
782
+ # so every backfill appended a second copy of every historical row.
783
+ if not cleared:
784
+ con.execute(f"DELETE FROM {mirror} WHERE 1=1")
785
+ cleared = True
786
  written += _replace(con, mirror, cols, types, rows)
787
+
788
+ operations = {}
789
+ if table == "pricing":
790
+ state_key = ENTITY_STATE_KEYS[table]
791
+ if not pricing_asins:
792
+ _record_state(con, state_key, "unproven", 0,
793
+ "Sales and Traffic returned no ASIN to test")
794
+ operations["getCompetitiveSummary"] = {
795
+ "state": "unproven", "responses": 0}
796
+ log(" pricing: getCompetitiveSummary UNPROVEN, no ASIN was returned")
797
+ else:
798
+ try:
799
+ responses = cx.competitive_summaries(fields, pricing_asins)
800
+ response_rows, updated = _apply_competitive(con, mirror, responses)
801
+ _record_state(con, state_key, "done", response_rows)
802
+ operations["getCompetitiveSummary"] = {
803
+ "state": "done", "responses": response_rows,
804
+ "mirror_rows_enriched": updated}
805
+ log(f" pricing: getCompetitiveSummary {response_rows:,} ASIN responses, "
806
+ f"{updated:,} mirror rows enriched")
807
+ except Exception as e: # noqa: BLE001
808
+ detail = f"{type(e).__name__}: {str(e)[:160]}"
809
+ _record_state(con, state_key, "refused", 0, detail)
810
+ operations["getCompetitiveSummary"] = {
811
+ "state": "refused", "reason": detail}
812
+ log(f" pricing: getCompetitiveSummary REFUSED: {str(e)[:160]}")
813
+
814
+ if table == "inventory":
815
+ state_key = ENTITY_STATE_KEYS[table]
816
+ try:
817
+ inventory = cx.awd_inventory(fields)
818
+ awd_rows = _awd_rows(inventory)
819
+ if awd_rows:
820
+ awd_cols = sorted({c for row in awd_rows for c in row})
821
+ awd_types = _ensure(con, mirror, awd_cols, awd_rows)
822
+ if not cleared:
823
+ con.execute(f"DELETE FROM {mirror} WHERE 1=1")
824
+ cleared = True
825
+ written += _replace(con, mirror, awd_cols, awd_types, awd_rows)
826
+ _record_state(con, state_key, "done", len(awd_rows))
827
+ operations["listInventory"] = {
828
+ "state": "done", "rows": len(awd_rows)}
829
+ log(f" inventory: AWD listInventory {len(awd_rows):,} rows")
830
+ except Exception as e: # noqa: BLE001
831
+ detail = f"{type(e).__name__}: {str(e)[:160]}"
832
+ _record_state(con, state_key, "refused", 0, detail)
833
+ operations["listInventory"] = {"state": "refused", "reason": detail}
834
+ log(f" inventory: AWD listInventory REFUSED: {str(e)[:160]}")
835
+
836
+ try:
837
+ column_count = len(con.execute(f"PRAGMA table_info('{mirror}')").fetchall())
838
+ total_rows = int(con.execute(f"SELECT COUNT(*) FROM {mirror}").fetchone()[0])
839
+ except Exception: # noqa: BLE001
840
+ column_count, total_rows = len(cols), written
841
+ report["tables"][table] = {
842
+ "rows": total_rows, "columns": column_count, "provenance": prov,
843
+ "report": rtype, "operations": operations}
844
+ _record_state(con, key, "done", total_rows)
845
+ log(f" {table:<16} {total_rows:>7,} rows {column_count:>3} cols {prov}")
846
  except Exception as e: # noqa: BLE001
847
  # ⚠ ONE TABLE'S REFUSAL IS NOT THE SYNC'S. A 403 on brand_analytics is R5's UNPROVEN
848
  # ticket answering itself, and it must not cost the six tables that did work.
 
889
  [[a-check-that-cannot-reach-green-is-broken]].
890
  """
891
  rt = rt if rt is not None else _runtime_for(tenant_key)
892
+ con, _path = _connect_tenant_mirror(tenant_key, rt)
893
  cx = _client()
894
  out = {}
895
  for table in TABLES:
 
904
  out[table] = {"state": "unproven", "reason": "table is empty",
905
  "reconciles": spec.get("validates_against")}
906
  continue
907
+ if table in ENTITY_STATE_KEYS:
908
+ operation = _read_state(con, ENTITY_STATE_KEYS[table])
909
+ if not operation or operation.get("state") != "done":
910
+ out[table] = {
911
+ "state": "unproven",
912
+ "reason": (operation or {}).get("detail")
913
+ or "required role operation has not completed",
914
+ "reconciles": spec.get("validates_against")}
915
+ continue
916
+ expected = int(operation.get("rows") or 0)
917
+ if table == "inventory":
918
+ measured = int(con.execute(
919
+ f"SELECT COUNT(*) FROM {mirror} WHERE inventory_source='AWD'").fetchone()[0])
920
+ else:
921
+ measured = int(con.execute(
922
+ f"SELECT COUNT(DISTINCT competitive_body_asin) FROM {mirror} "
923
+ "WHERE competitive_body_asin IS NOT NULL").fetchone()[0])
924
+ if measured != expected:
925
+ out[table] = {
926
+ "state": "refused", "rows": n,
927
+ "reason": f"role operation returned {expected} but mirror carries {measured}",
928
+ "reconciles": spec.get("validates_against")}
929
+ continue
930
  out[table] = {"state": "ok", "rows": n, "reconciles": spec.get("validates_against")}
931
  for t, r in out.items():
932
  log(f" {t:<16} {r['state']:<10} {r.get('rows', 0):>7,} {r.get('reconciles') or ''}")
 
937
  """What is in the mirror now. Needs no credential and no network."""
938
  rt = rt if rt is not None else _runtime_for(tenant_key)
939
  try:
940
+ con, _path = _connect_tenant_mirror(tenant_key, rt)
941
  except Exception as e: # noqa: BLE001
942
  return {"error": f"{type(e).__name__}: {e}"}
943
  out = {}
 
950
  n, cols = None, 0
951
  out[table] = {"rows": n, "columns": cols,
952
  "provenance": amazon_schema.PROVENANCE.get(table)}
953
+ if table in ENTITY_STATE_KEYS:
954
+ out[table]["operation"] = _read_state(con, ENTITY_STATE_KEYS[table])
955
  return out
956
 
957
 
 
1076
  ck("NC-AZ15 an empty document yields no columns and no rows",
1077
  parse_flat_file("") == ([], []) and parse_json_report("", "x") == ([], []))
1078
 
1079
+ # NC-AZ16..18 - both role-specific payloads keep their complete source shape and source label.
1080
+ price_input = [
1081
+ {"child_asin": "B01", "parent_asin": "P01"},
1082
+ {"child_asin": "B01", "parent_asin": "P01"},
1083
+ {"parent_asin": "P02"},
1084
+ ]
1085
+ ck("NC-AZ16 pricing ASINs are complete and de-duplicated",
1086
+ _pricing_asins(price_input) == ["B01", "P02"], _pricing_asins(price_input))
1087
+ competitive = [{
1088
+ "status": {"statusCode": 200, "reasonPhrase": "Success"},
1089
+ "body": {"asin": "B01", "marketplaceId": "ATVPDKIKX0DER",
1090
+ "featuredBuyingOptions": [{"listingPrice": {"amount": 12.5}}],
1091
+ "referencePrices": [{"name": "CompetitivePrice", "price": {"amount": 11.0}}]},
1092
+ }]
1093
+ cr = _competitive_rows(competitive)
1094
+ ck("NC-AZ17 competitive summary keeps status and every returned data family",
1095
+ bool(cr) and cr[0].get("competitive_status_status_code") == 200
1096
+ and "competitive_body_featured_buying_options" in cr[0]
1097
+ and "competitive_body_reference_prices" in cr[0], cr)
1098
+ awd = _awd_rows([{"sku": "RI-1", "totalOnhandQuantity": 9,
1099
+ "inventoryDetails": {"replenishmentQuantity": 2}}])
1100
+ ck("NC-AZ18 AWD inventory remains source-labelled and nested fields survive",
1101
+ bool(awd) and awd[0].get("inventory_source") == "AWD"
1102
+ and awd[0].get("inventory_details_replenishment_quantity") == 2, awd)
1103
+
1104
+ import duckdb
1105
+ mem = duckdb.connect(":memory:")
1106
+ mem.execute("CREATE TABLE amzn_pricing (child_asin VARCHAR, parent_asin VARCHAR)")
1107
+ mem.execute("INSERT INTO amzn_pricing VALUES ('B01','P01'),('B01','P01')")
1108
+ response_count, updated = _apply_competitive(mem, "amzn_pricing", competitive)
1109
+ enriched = mem.execute(
1110
+ "SELECT COUNT(*) FROM amzn_pricing WHERE competitive_body_asin='B01'").fetchone()[0]
1111
+ ck("NC-AZ19 competitive summary enriches every matching report row without appending",
1112
+ response_count == 1 and updated == 2 and enriched == 2,
1113
+ (response_count, updated, enriched))
1114
+
1115
+ mem.execute("""CREATE TABLE _sync_state (
1116
+ entity VARCHAR PRIMARY KEY, phase VARCHAR, last_id BIGINT,
1117
+ cursor_wd VARCHAR, rows BIGINT, updated_at VARCHAR)""")
1118
+ _record_state(mem, "amzn_inventory_awd", "done", 7)
1119
+ marker = mem.execute(
1120
+ "SELECT phase, last_id, cursor_wd, rows, updated_at FROM _sync_state").fetchone()
1121
+ ck("NC-AZ20 sync state writes rows and timestamp into their declared columns",
1122
+ marker[0] == "done" and marker[1] is None and marker[2] is None
1123
+ and marker[3] == 7 and bool(marker[4]), marker)
1124
+
1125
+ # NC-AZ21..24 - the destructive path is hermetic, prefix-complete, and collateral-free.
1126
+ mem.execute("CREATE TABLE amzn_orders (id BIGINT)")
1127
+ mem.execute("INSERT INTO amzn_orders VALUES (1), (2)")
1128
+ mem.execute("CREATE TABLE amzn_future_table (id BIGINT)")
1129
+ mem.execute("INSERT INTO amzn_future_table VALUES (3)")
1130
+ mem.execute("CREATE TABLE keep_unrelated (id BIGINT)")
1131
+ mem.execute("INSERT INTO keep_unrelated VALUES (99)")
1132
+ _record_state(mem, "amzn_orders", "done", 2)
1133
+ _record_state(mem, "odoo_orders", "live", 11)
1134
+ purged = _purge_mirror(mem)
1135
+ ck("NC-AZ21 purge removes every amzn_ table, including a future one",
1136
+ purged["tables"] == ["amzn_future_table", "amzn_orders", "amzn_pricing"]
1137
+ and purged["rows"] == 5, purged)
1138
+ ck("NC-AZ22 purge removes every Amazon sync marker",
1139
+ purged["syncMarkers"] == 2
1140
+ and not [r for r in _sync_state_rows(mem) if str(r[0]).startswith("amzn_")],
1141
+ _sync_state_rows(mem))
1142
+ kept = mem.execute("SELECT id FROM keep_unrelated").fetchall()
1143
+ kept_state = mem.execute(
1144
+ "SELECT phase, rows FROM _sync_state WHERE entity='odoo_orders'").fetchone()
1145
+ ck("NC-AZ23 purge preserves unrelated tables and sync markers byte for byte",
1146
+ kept == [(99,)] and kept_state == ("live", 11), (kept, kept_state))
1147
+ second = _purge_mirror(mem)
1148
+ ck("NC-AZ24 purge readback is residue-free and a retry is idempotent",
1149
+ second == {"tables": [], "rows": 0, "syncMarkers": 0,
1150
+ "residueTables": [], "residueSyncMarkers": []}, second)
1151
+
1152
+ # NC-AZ25..26 - neither a typo nor a still-live credential can reach the DROP transaction.
1153
+ refused_confirmation = False
1154
+ try:
1155
+ _require_purge_confirmation("royal-imports", "royal_imports")
1156
+ except RuntimeError:
1157
+ refused_confirmation = True
1158
+ ck("NC-AZ25 purge refuses when the exact tenant confirmation mismatches",
1159
+ refused_confirmation)
1160
+
1161
+ class _FakeRuntime:
1162
+ def __init__(self, entries):
1163
+ self.entries = entries
1164
+
1165
+ def get(self, name):
1166
+ from core import keychain as _kc
1167
+ return {"entries": self.entries} if name == _kc.KEY else None
1168
+
1169
+ absent_ok, present_refused = False, False
1170
+ try:
1171
+ absent_ok = _assert_amazon_credential_revoked(_FakeRuntime({}))
1172
+ _assert_amazon_credential_revoked(_FakeRuntime({
1173
+ "k_probe": {"type": "amazon_sp", "label": "Amazon",
1174
+ "enc": "not-decrypted-by-this-check"},
1175
+ }))
1176
+ except RuntimeError:
1177
+ present_refused = True
1178
+ ck("NC-AZ26 purge refuses while any Amazon credential metadata remains",
1179
+ absent_ok and present_refused, (absent_ok, present_refused))
1180
+ mem.close()
1181
+
1182
+ # NC-AZ27 - drive the public operator wrapper against an isolated file, including its readback.
1183
+ import tempfile
1184
+ prior_path = Path(datastore.DB_PATH)
1185
+ public_ok, public_detail = False, None
1186
+ try:
1187
+ with tempfile.TemporaryDirectory() as tmp:
1188
+ purge_path = Path(tmp) / "qa-purge.duckdb"
1189
+ seed = duckdb.connect(str(purge_path))
1190
+ seed.execute("""CREATE TABLE _sync_state (
1191
+ entity VARCHAR PRIMARY KEY, phase VARCHAR, last_id BIGINT,
1192
+ cursor_wd VARCHAR, rows BIGINT, updated_at VARCHAR)""")
1193
+ seed.execute("CREATE TABLE amzn_orders (id BIGINT)")
1194
+ seed.execute("INSERT INTO amzn_orders VALUES (1)")
1195
+ _record_state(seed, "amzn_orders", "done", 1)
1196
+ seed.close()
1197
+
1198
+ class _PurgeRuntime(_FakeRuntime):
1199
+ key = "qa-purge"
1200
+ datastore_path = purge_path
1201
+
1202
+ def assert_datastore_matches(self):
1203
+ if Path(datastore.DB_PATH).resolve() != purge_path.resolve():
1204
+ raise RuntimeError("wrong test mirror")
1205
+
1206
+ public_detail = purge(
1207
+ "qa-purge", "qa-purge", rt=_PurgeRuntime({}), log=lambda *_a: None)
1208
+ datastore.use_path(prior_path)
1209
+ readback = duckdb.connect(str(purge_path), read_only=True)
1210
+ tables_after = sorted(str(row[0]) for row in readback.execute("SHOW TABLES").fetchall())
1211
+ state_after = readback.execute("SELECT entity FROM _sync_state").fetchall()
1212
+ readback.close()
1213
+ public_ok = (public_detail["tenant"] == "qa-purge"
1214
+ and public_detail["database"] == "qa-purge.duckdb"
1215
+ and public_detail["rows"] == 1
1216
+ and tables_after == ["_sync_state"] and state_after == [])
1217
+ finally:
1218
+ datastore.use_path(prior_path)
1219
+ ck("NC-AZ27 the public purge wrapper deletes and reads back an isolated tenant file",
1220
+ public_ok, public_detail)
1221
+
1222
+ log(f"\n{'GREEN' if not fails else 'RED'}: {27 - len(fails)}/27")
1223
  return 0 if not fails else 1
1224
 
1225
 
 
1228
  ap.add_argument("--sync", action="store_true")
1229
  ap.add_argument("--status", action="store_true")
1230
  ap.add_argument("--validate", action="store_true")
1231
+ ap.add_argument("--purge", action="store_true",
1232
+ help="permanently delete every Amazon mirror table after credential revocation")
1233
  ap.add_argument("--selftest", action="store_true")
1234
+ ap.add_argument("--tenant", default=None)
1235
+ ap.add_argument("--confirm-tenant", default=None,
1236
+ help="for --purge, repeat the exact tenant slug")
1237
  ap.add_argument("--table", action="append", help="limit to one table (repeatable)")
1238
  a = ap.parse_args(argv)
1239
+ tenant = a.tenant or "royal-imports"
1240
+ if a.purge:
1241
+ if not a.tenant:
1242
+ ap.error("--purge requires an explicit --tenant")
1243
+ if a.table or a.sync or a.status or a.validate or a.selftest:
1244
+ ap.error("--purge is all-or-nothing and cannot be combined with another action")
1245
+ rep = purge(a.tenant, a.confirm_tenant)
1246
+ print(json.dumps(rep, indent=2))
1247
+ return 0
1248
  if a.selftest:
1249
  return selftest()
1250
  if a.status:
1251
+ print(json.dumps(status(tenant), indent=2))
1252
  return 0
1253
  if a.validate:
1254
+ validate(tenant)
1255
  return 0
1256
  if a.sync:
1257
  t0 = time.time()
1258
+ rep = sync(tenant, tables=a.table)
1259
  print(json.dumps(rep, indent=2))
1260
  print(f"\n{time.time() - t0:.1f}s")
1261
  return 0
requirements.txt CHANGED
@@ -15,8 +15,9 @@
15
  # wave 21 (C5) as a LAZY dep of routes_uploads.py β€” the boot proof above still holds because the
16
  # import lives inside the handler, and the gate now asserts this line exists here.
17
  fastapi>=0.139
 
18
  uvicorn[standard]>=0.30
19
- python-dotenv>=1.0
20
  pandas>=2.0
21
  requests>=2.28
22
  huggingface_hub>=0.20
@@ -44,7 +45,7 @@ openpyxl>=3.1
44
  # python-multipart AT IMPORT TIME for any route declaring File(...)/Form(...). Every local gate
45
  # was green because the dev box happens to have it for unrelated reasons β€” the container
46
  # installs exactly this file. The environment-parity twin of the streamlit lesson above.
47
- python-multipart>=0.0.20
48
  # ⭐ WAVE 37 (D-346, answering `ASK E-1`) β€” THE OFFICIAL ANTHROPIC SDK. `ai_review.py` called the
49
  # Messages API over raw `requests`, which meant re-implementing retries, streaming and error
50
  # taxonomy by hand and drifting from them silently. E ships the SDK behind a LAZY import with the
 
15
  # wave 21 (C5) as a LAZY dep of routes_uploads.py β€” the boot proof above still holds because the
16
  # import lives inside the handler, and the gate now asserts this line exists here.
17
  fastapi>=0.139
18
+ starlette>=1.3.1
19
  uvicorn[standard]>=0.30
20
+ python-dotenv>=1.2.2
21
  pandas>=2.0
22
  requests>=2.28
23
  huggingface_hub>=0.20
 
45
  # python-multipart AT IMPORT TIME for any route declaring File(...)/Form(...). Every local gate
46
  # was green because the dev box happens to have it for unrelated reasons β€” the container
47
  # installs exactly this file. The environment-parity twin of the streamlit lesson above.
48
+ python-multipart>=0.0.31
49
  # ⭐ WAVE 37 (D-346, answering `ASK E-1`) β€” THE OFFICIAL ANTHROPIC SDK. `ai_review.py` called the
50
  # Messages API over raw `requests`, which meant re-implementing retries, streaming and error
51
  # taxonomy by hand and drifting from them silently. E ships the SDK behind a LAZY import with the