Deploy AIOS web (React glide grid + FastAPI slice)
Browse files- RELEASES.json +1 -1
- VERSION +1 -1
- api/main.py +55 -4
- api/odoo_relational.py +13 -4
- api/routes_nav.py +68 -3
- api/routes_products.py +16 -3
- api/routes_sheet.py +124 -4
- api/routes_starred.py +2 -1
- platform/core/audit.py +127 -24
- platform/harness/amazon_schema.py +2 -1
- platform/harness/amazon_store.py +264 -5
- platform/harness/mirror_store.py +13 -0
- platform/modules/opportunity_finder.py +192 -33
- web/public/AIOS_DEPLOY_RECEIPT +1 -1
- web/src/apps/catalog/CatalogApp.tsx +5 -4
- web/src/customer-grid/CatalogView.tsx +26 -6
RELEASES.json
CHANGED
|
@@ -1,4 +1,4 @@
|
|
| 1 |
{
|
| 2 |
-
"current": "
|
| 3 |
"releases": []
|
| 4 |
}
|
|
|
|
| 1 |
{
|
| 2 |
+
"current": "f76135962",
|
| 3 |
"releases": []
|
| 4 |
}
|
VERSION
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
|
|
|
|
| 1 |
+
f76135962
|
api/main.py
CHANGED
|
@@ -2267,8 +2267,10 @@ def _resync_mirror_pass(manual, passes, entities=None):
|
|
| 2267 |
print(f"[aios-api] store resync: the absent tables first failed "
|
| 2268 |
f"({type(exc).__name__}); the full pass below still syncs them")
|
| 2269 |
_resync_log("full pass start")
|
|
|
|
| 2270 |
sync_report = mirror_store.tracked_sync("odoo", _ds._resident_odoo_runtime(), _ds.sync_all,
|
| 2271 |
-
entities=entities, log=_resync_log
|
|
|
|
| 2272 |
_log_entities("full", sync_report)
|
| 2273 |
for key, row in first.items():
|
| 2274 |
if isinstance(sync_report, dict) and isinstance(sync_report.get(key), dict):
|
|
@@ -2463,20 +2465,69 @@ def _resync_confirm_fresh(tenant, cadence_seconds):
|
|
| 2463 |
print(f"[aios-api] odoo freshness stamp not saved ({type(e).__name__}: {e})")
|
| 2464 |
|
| 2465 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2466 |
def _capture_resync_source_counts(tenant):
|
| 2467 |
"""Count the default connection's Odoo source right after its mirror pull, or None.
|
| 2468 |
|
| 2469 |
The vendor pulls run between the mirror pull and the rebuild, so a count taken at rebuild
|
| 2470 |
-
time describes a later source than the mirror holds (D-9131).
|
|
|
|
| 2471 |
"""
|
| 2472 |
try:
|
| 2473 |
if not tenant:
|
| 2474 |
return None
|
| 2475 |
import odoo_relational as _rel_counts
|
| 2476 |
from harness import runtime as _runtime_counts
|
| 2477 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2478 |
print(f"[aios-api] store resync: counted the Odoo source for {len(counts)} "
|
| 2479 |
-
f"database(s) at mirror-pull time", flush=True)
|
| 2480 |
return counts or None
|
| 2481 |
except Exception as e: # noqa: BLE001
|
| 2482 |
print(f"[aios-api] store resync: the Odoo source was not counted at mirror-pull time "
|
|
|
|
| 2267 |
print(f"[aios-api] store resync: the absent tables first failed "
|
| 2268 |
f"({type(exc).__name__}); the full pass below still syncs them")
|
| 2269 |
_resync_log("full pass start")
|
| 2270 |
+
_after_entity = _entity_source_counter(_ds)
|
| 2271 |
sync_report = mirror_store.tracked_sync("odoo", _ds._resident_odoo_runtime(), _ds.sync_all,
|
| 2272 |
+
entities=entities, log=_resync_log,
|
| 2273 |
+
**({"after_entity": _after_entity} if _after_entity else {}))
|
| 2274 |
_log_entities("full", sync_report)
|
| 2275 |
for key, row in first.items():
|
| 2276 |
if isinstance(sync_report, dict) and isinstance(sync_report.get(key), dict):
|
|
|
|
| 2465 |
print(f"[aios-api] odoo freshness stamp not saved ({type(e).__name__}: {e})")
|
| 2466 |
|
| 2467 |
|
| 2468 |
+
#: tenant -> {bucket: (count, mode)} counted beside each entity's copy during the last mirror pass.
|
| 2469 |
+
_PASS_SOURCE_COUNTS = {}
|
| 2470 |
+
|
| 2471 |
+
|
| 2472 |
+
def _entity_source_counter(ds):
|
| 2473 |
+
"""A `sync_all` hook that counts an entity's source buckets right after its copy, or None.
|
| 2474 |
+
|
| 2475 |
+
None when `sync_all` takes no `after_entity` hook: the loop then counts once after the pass.
|
| 2476 |
+
"""
|
| 2477 |
+
import inspect as _inspect_counts
|
| 2478 |
+
try:
|
| 2479 |
+
if "after_entity" not in _inspect_counts.signature(ds.sync_all).parameters:
|
| 2480 |
+
return None
|
| 2481 |
+
rt = ds._resident_odoo_runtime()
|
| 2482 |
+
except Exception: # noqa: BLE001
|
| 2483 |
+
return None
|
| 2484 |
+
tenant = getattr(rt, "key", None)
|
| 2485 |
+
if not tenant:
|
| 2486 |
+
return None
|
| 2487 |
+
counted = _PASS_SOURCE_COUNTS[tenant] = {}
|
| 2488 |
+
|
| 2489 |
+
def after_entity(key):
|
| 2490 |
+
# A volatile table (stock_quant) drifts between its copy and a count at the end of the pass.
|
| 2491 |
+
try:
|
| 2492 |
+
import odoo_relational as _rel_counts
|
| 2493 |
+
model = (ds.ENTITIES.get(key) or {}).get("model")
|
| 2494 |
+
buckets = _rel_counts.source_count_buckets(model) - set(counted)
|
| 2495 |
+
if not buckets:
|
| 2496 |
+
return
|
| 2497 |
+
got = _rel_counts.capture_source_counts(rt, buckets=buckets)
|
| 2498 |
+
counted.update(got)
|
| 2499 |
+
for bucket, (count, _mode) in sorted(got.items()):
|
| 2500 |
+
print(f"[aios-api] store resync: counted the Odoo source for {bucket} "
|
| 2501 |
+
f"beside the {key} copy: {count:,} rows", flush=True)
|
| 2502 |
+
except Exception as e: # noqa: BLE001
|
| 2503 |
+
print(f"[aios-api] store resync: the {key} source was not counted beside its copy "
|
| 2504 |
+
f"({type(e).__name__}); it is counted after the pass", flush=True)
|
| 2505 |
+
return after_entity
|
| 2506 |
+
|
| 2507 |
+
|
| 2508 |
def _capture_resync_source_counts(tenant):
|
| 2509 |
"""Count the default connection's Odoo source right after its mirror pull, or None.
|
| 2510 |
|
| 2511 |
The vendor pulls run between the mirror pull and the rebuild, so a count taken at rebuild
|
| 2512 |
+
time describes a later source than the mirror holds (D-9131). A bucket counted beside its
|
| 2513 |
+
entity's copy during the pass keeps that count. Never raises.
|
| 2514 |
"""
|
| 2515 |
try:
|
| 2516 |
if not tenant:
|
| 2517 |
return None
|
| 2518 |
import odoo_relational as _rel_counts
|
| 2519 |
from harness import runtime as _runtime_counts
|
| 2520 |
+
during = dict((globals().get("_PASS_SOURCE_COUNTS") or {}).pop(tenant, None) or {})
|
| 2521 |
+
rt = _runtime_counts.get_runtime(tenant)
|
| 2522 |
+
counts = dict(during)
|
| 2523 |
+
if not during:
|
| 2524 |
+
counts.update(_rel_counts.capture_source_counts(rt))
|
| 2525 |
+
else:
|
| 2526 |
+
rest = _rel_counts.enabled_buckets(rt) - set(during)
|
| 2527 |
+
if rest:
|
| 2528 |
+
counts.update(_rel_counts.capture_source_counts(rt, buckets=rest))
|
| 2529 |
print(f"[aios-api] store resync: counted the Odoo source for {len(counts)} "
|
| 2530 |
+
f"database(s) at mirror-pull time, {len(during)} beside their own copy", flush=True)
|
| 2531 |
return counts or None
|
| 2532 |
except Exception as e: # noqa: BLE001
|
| 2533 |
print(f"[aios-api] store resync: the Odoo source was not counted at mirror-pull time "
|
api/odoo_relational.py
CHANGED
|
@@ -3914,12 +3914,18 @@ def _source_domain_count(client, bucket):
|
|
| 3914 |
raise Refused("the database source-count mode is not supported")
|
| 3915 |
|
| 3916 |
|
| 3917 |
-
def
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3918 |
"""`{bucket: (count, mode)}` for each enabled curated database, counted at the source now.
|
| 3919 |
|
| 3920 |
The resync loop calls this right after the mirror pull, so reconciliation compares the mirror
|
| 3921 |
-
with the source as of the same moment.
|
| 3922 |
-
bucket counts fresh at reconciliation. No source or
|
|
|
|
| 3923 |
"""
|
| 3924 |
try:
|
| 3925 |
source = (rt.odoo_source() if connection_id is None else rt.odoo_source(connection_id))
|
|
@@ -3936,7 +3942,10 @@ def capture_source_counts(rt, connection_id=None):
|
|
| 3936 |
return {}
|
| 3937 |
if client is None:
|
| 3938 |
return {}
|
| 3939 |
-
|
|
|
|
|
|
|
|
|
|
| 3940 |
try:
|
| 3941 |
counts[bucket] = _source_domain_count(client, bucket)
|
| 3942 |
except Exception: # noqa: BLE001
|
|
|
|
| 3914 |
raise Refused("the database source-count mode is not supported")
|
| 3915 |
|
| 3916 |
|
| 3917 |
+
def source_count_buckets(model):
|
| 3918 |
+
"""The curated buckets whose source population is counted on one Odoo model."""
|
| 3919 |
+
return {bucket for bucket, spec in _SOURCE_COUNT_SPECS.items() if spec["model"] == model}
|
| 3920 |
+
|
| 3921 |
+
|
| 3922 |
+
def capture_source_counts(rt, connection_id=None, buckets=None):
|
| 3923 |
"""`{bucket: (count, mode)}` for each enabled curated database, counted at the source now.
|
| 3924 |
|
| 3925 |
The resync loop calls this right after the mirror pull, so reconciliation compares the mirror
|
| 3926 |
+
with the source as of the same moment. `buckets` limits the count to those buckets. A bucket
|
| 3927 |
+
whose count raises is left out, and that bucket counts fresh at reconciliation. No source or
|
| 3928 |
+
no client returns `{}`.
|
| 3929 |
"""
|
| 3930 |
try:
|
| 3931 |
source = (rt.odoo_source() if connection_id is None else rt.odoo_source(connection_id))
|
|
|
|
| 3942 |
return {}
|
| 3943 |
if client is None:
|
| 3944 |
return {}
|
| 3945 |
+
chosen = enabled_buckets(rt, connection_id)
|
| 3946 |
+
if buckets is not None:
|
| 3947 |
+
chosen = chosen & set(buckets)
|
| 3948 |
+
for bucket in sorted(chosen):
|
| 3949 |
try:
|
| 3950 |
counts[bucket] = _source_domain_count(client, bucket)
|
| 3951 |
except Exception: # noqa: BLE001
|
api/routes_nav.py
CHANGED
|
@@ -459,11 +459,39 @@ def _visible_database_keys(session):
|
|
| 459 |
return keys, enumerated
|
| 460 |
|
| 461 |
|
| 462 |
-
def
|
| 463 |
-
"""`
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 464 |
|
| 465 |
`owner` may be ``None`` for a legacy ownerless grant. Callers preserve the old fail-closed
|
| 466 |
source lookup for that case; modern grants point directly at the canonical personal stratum.
|
|
|
|
|
|
|
| 467 |
"""
|
| 468 |
import core.shares as shares
|
| 469 |
try:
|
|
@@ -476,9 +504,46 @@ def _granted_view_sources(session):
|
|
| 476 |
# add a View to "shared with me"; a valid named or everyone grant did.
|
| 477 |
if shares.role_from(record, session.uname) is not None:
|
| 478 |
out[str(raw_id)] = owner
|
| 479 |
-
return out
|
| 480 |
except Exception:
|
| 481 |
return {}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 482 |
|
| 483 |
|
| 484 |
def _granted_view_ids(session):
|
|
|
|
| 459 |
return keys, enumerated
|
| 460 |
|
| 461 |
|
| 462 |
+
def _folder_views_in(doc, folder_id, owner=None):
|
| 463 |
+
"""`(owner, [view_id])` for the VIEWS folder `folder_id` in one held workspace doc, else None.
|
| 464 |
+
|
| 465 |
+
The same lookup as `TableStore.find_folder`, over a projection the caller already read.
|
| 466 |
+
"""
|
| 467 |
+
import core.table_store as table_store
|
| 468 |
+
fid = str(folder_id or "").strip()
|
| 469 |
+
if not fid or not isinstance(doc, dict):
|
| 470 |
+
return None
|
| 471 |
+
for who, ws in doc.items():
|
| 472 |
+
if who in table_store.RESERVED_MEMBERS or not isinstance(ws, dict):
|
| 473 |
+
continue
|
| 474 |
+
if owner and str(who).strip().lower() != owner:
|
| 475 |
+
continue
|
| 476 |
+
rows = (ws.get("folders") or {}).get("views") or []
|
| 477 |
+
if not any(isinstance(f, dict) and str(f.get("id") or "") == fid for f in rows):
|
| 478 |
+
continue
|
| 479 |
+
# `itemFolders` is keyed by surface first: the views map sits under 'views'.
|
| 480 |
+
placed = (ws.get("itemFolders") or {}).get("views") or {}
|
| 481 |
+
placed = placed if isinstance(placed, dict) else {}
|
| 482 |
+
views = ws.get("views") or {}
|
| 483 |
+
return str(who), [str(vid) for vid, view in views.items()
|
| 484 |
+
if isinstance(view, dict) and str(placed.get(str(vid)) or "") == fid]
|
| 485 |
+
return None
|
| 486 |
+
|
| 487 |
+
|
| 488 |
+
def _granted_view_sources(session, docs=None):
|
| 489 |
+
"""`{view_id: owner}` for grants this caller may read, from one registry read per kind.
|
| 490 |
|
| 491 |
`owner` may be ``None`` for a legacy ownerless grant. Callers preserve the old fail-closed
|
| 492 |
source lookup for that case; modern grants point directly at the canonical personal stratum.
|
| 493 |
+
`docs` is `{database: projected workspace doc}` the caller already holds; the folder leg
|
| 494 |
+
reads views from it, else from `TableStore.find_folder` per visible database.
|
| 495 |
"""
|
| 496 |
import core.shares as shares
|
| 497 |
try:
|
|
|
|
| 504 |
# add a View to "shared with me"; a valid named or everyone grant did.
|
| 505 |
if shares.role_from(record, session.uname) is not None:
|
| 506 |
out[str(raw_id)] = owner
|
|
|
|
| 507 |
except Exception:
|
| 508 |
return {}
|
| 509 |
+
# Folder leg (the `grid_events._granted_views` D-37 rule): a View filed in a folder shared
|
| 510 |
+
# with the caller is shared with the caller. Direct grants win, so this leg only fills gaps.
|
| 511 |
+
# A failure here drops the folder leg alone and keeps the direct answer: narrower, never wider.
|
| 512 |
+
try:
|
| 513 |
+
folders = {}
|
| 514 |
+
for raw_fid, record in shares.grants_bulk("folder", st=session.runtime).items():
|
| 515 |
+
owner = str(record.get("owner") or "").strip().lower() or None
|
| 516 |
+
if owner == session.uname:
|
| 517 |
+
continue
|
| 518 |
+
if shares.role_from(record, session.uname) in ("view", "edit"):
|
| 519 |
+
folders[str(raw_fid)] = owner
|
| 520 |
+
if not folders:
|
| 521 |
+
return out
|
| 522 |
+
hits = []
|
| 523 |
+
if docs is not None:
|
| 524 |
+
for doc in docs.values():
|
| 525 |
+
for fid, owner in folders.items():
|
| 526 |
+
hits.append(_folder_views_in(doc, fid, owner))
|
| 527 |
+
else:
|
| 528 |
+
import core.table_store as table_store
|
| 529 |
+
import core.view_templates as view_templates
|
| 530 |
+
keys, _ = _visible_database_keys(session)
|
| 531 |
+
for key in sorted(keys):
|
| 532 |
+
ws_key = view_templates.workspace_key(key)
|
| 533 |
+
if not ws_key:
|
| 534 |
+
continue
|
| 535 |
+
tops = table_store.make(ws_key, st=session.runtime)
|
| 536 |
+
for fid in folders:
|
| 537 |
+
found = tops.find_folder(fid)
|
| 538 |
+
hits.append((found[0], list(found[2])) if found else None)
|
| 539 |
+
for hit in hits:
|
| 540 |
+
if not hit or str(hit[0]).strip().lower() == session.uname:
|
| 541 |
+
continue
|
| 542 |
+
for vid in hit[1]:
|
| 543 |
+
out.setdefault(str(vid), hit[0])
|
| 544 |
+
except Exception:
|
| 545 |
+
pass
|
| 546 |
+
return out
|
| 547 |
|
| 548 |
|
| 549 |
def _granted_view_ids(session):
|
api/routes_products.py
CHANGED
|
@@ -39,6 +39,8 @@ router = APIRouter(prefix="/api/v1")
|
|
| 39 |
MODULE = "product_data"
|
| 40 |
|
| 41 |
_CACHE_TTL = 900
|
|
|
|
|
|
|
| 42 |
|
| 43 |
|
| 44 |
def _pool_for(rt, team_id):
|
|
@@ -105,14 +107,25 @@ def _pool_for(rt, team_id):
|
|
| 105 |
# keeping existing adapters valid, this avoids paying for an unnecessary `extra_fields`
|
| 106 |
# branch on the overwhelmingly common/default-off path.
|
| 107 |
from harness import canonical_source, runtime as _runtime
|
|
|
|
| 108 |
# Stock, sales and product reads come from the Odoo mirror; an unsynced mirror raises.
|
| 109 |
with canonical_source.mirrored(lambda: _runtime.mirror_cursor(rt).cursor()):
|
| 110 |
rows = (pd.pool(team_id=team_id,
|
| 111 |
raw_fields=[field.get("rawName") for field in enabled_raw])
|
| 112 |
if enabled_raw else pd.pool(team_id=team_id))
|
| 113 |
-
|
| 114 |
-
|
| 115 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 116 |
|
| 117 |
|
| 118 |
def scoped_pool(session: Session, *, with_rows: bool = True):
|
|
|
|
| 39 |
MODULE = "product_data"
|
| 40 |
|
| 41 |
_CACHE_TTL = 900
|
| 42 |
+
#: A product pool request slower than this logs its wait, so a cold scope is measurable on staging.
|
| 43 |
+
_SLOW_POOL_S = 5.0
|
| 44 |
|
| 45 |
|
| 46 |
def _pool_for(rt, team_id):
|
|
|
|
| 107 |
# keeping existing adapters valid, this avoids paying for an unnecessary `extra_fields`
|
| 108 |
# branch on the overwhelmingly common/default-off path.
|
| 109 |
from harness import canonical_source, runtime as _runtime
|
| 110 |
+
started = time.perf_counter()
|
| 111 |
# Stock, sales and product reads come from the Odoo mirror; an unsynced mirror raises.
|
| 112 |
with canonical_source.mirrored(lambda: _runtime.mirror_cursor(rt).cursor()):
|
| 113 |
rows = (pd.pool(team_id=team_id,
|
| 114 |
raw_fields=[field.get("rawName") for field in enabled_raw])
|
| 115 |
if enabled_raw else pd.pool(team_id=team_id))
|
| 116 |
+
projected = _project_enabled_raw_rows(rows, enabled_raw, drop_source=True)
|
| 117 |
+
# The cold path's own receipt in the Space log: the build alone, apart from any wait.
|
| 118 |
+
print(f"[aios-api] product pool built team={team_id} rows={len(projected)} "
|
| 119 |
+
f"in {time.perf_counter() - started:.1f}s", flush=True)
|
| 120 |
+
return projected
|
| 121 |
+
|
| 122 |
+
asked = time.perf_counter()
|
| 123 |
+
served = scope_cache.get(rt.pool_cache, key, _CACHE_TTL, build)
|
| 124 |
+
waited = time.perf_counter() - asked
|
| 125 |
+
if waited >= _SLOW_POOL_S:
|
| 126 |
+
# A wait far past the build line above is another build of this scope holding the lock.
|
| 127 |
+
print(f"[aios-api] product pool served team={team_id} after {waited:.1f}s", flush=True)
|
| 128 |
+
return served
|
| 129 |
|
| 130 |
|
| 131 |
def scoped_pool(session: Session, *, with_rows: bool = True):
|
api/routes_sheet.py
CHANGED
|
@@ -207,6 +207,119 @@ def _binding_catching_up(session, binding):
|
|
| 207 |
return any(_database_catching_up(session, target) for target in targets)
|
| 208 |
|
| 209 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 210 |
def _mirror(session: Session):
|
| 211 |
"""`(cursor, spec, columns)` for one read-through database, or the refusal that names why.
|
| 212 |
|
|
@@ -278,7 +391,7 @@ def _mirror(session: Session):
|
|
| 278 |
session._sheet_amzn_cursor = (cur, None)
|
| 279 |
else:
|
| 280 |
cur, mismatch = (getattr(session, "_sheet_drill_cursor", None)
|
| 281 |
-
or
|
| 282 |
except RuntimeError as exc:
|
| 283 |
raise err(503, "store_not_ready", str(exc))
|
| 284 |
if mismatch:
|
|
@@ -350,6 +463,10 @@ def _mirror(session: Session):
|
|
| 350 |
session._sheet_mirrors = memo
|
| 351 |
slot = (table_key, json.dumps(metric_window, sort_keys=True, default=str))
|
| 352 |
want = None if needed is None else {str(k) for k in needed}
|
|
|
|
|
|
|
|
|
|
|
|
|
| 353 |
entry = memo.get(slot)
|
| 354 |
if entry is not None:
|
| 355 |
built, made = entry
|
|
@@ -357,10 +474,10 @@ def _mirror(session: Session):
|
|
| 357 |
return made
|
| 358 |
if want is not None:
|
| 359 |
want = set(want) | built
|
| 360 |
-
made =
|
| 361 |
memo[slot] = (None if want is None else set(want), made)
|
| 362 |
return made
|
| 363 |
-
spec =
|
| 364 |
if spec is None and connector_defn is not None and str(table_key).startswith("ut_amzn_"):
|
| 365 |
spec = _amazon_stored_spec(cur, table_key)
|
| 366 |
if spec is None:
|
|
@@ -4449,6 +4566,7 @@ def _resolve_refresh_sheet(workbook_id: str, session: Session, *, only=None, for
|
|
| 4449 |
return hit["cell"]
|
| 4450 |
return None
|
| 4451 |
|
|
|
|
| 4452 |
started = perf_counter()
|
| 4453 |
resolved_count = 0
|
| 4454 |
held_refusal = None
|
|
@@ -5564,6 +5682,7 @@ _BINDING_REFUSAL_DISPLAY = {
|
|
| 5564 |
"store_not_ready": "#UPDATING",
|
| 5565 |
"mirror_not_ready": "#UPDATING",
|
| 5566 |
"amazon_mirror_not_loaded": "#UPDATING",
|
|
|
|
| 5567 |
}
|
| 5568 |
|
| 5569 |
|
|
@@ -5572,6 +5691,7 @@ _TRANSIENT_REFUSAL_CODES = frozenset((
|
|
| 5572 |
"store_catching_up", "source_unavailable", "store_not_ready", "mirror_not_ready",
|
| 5573 |
"mirror_mismatch", "amazon_mirror_not_loaded", "sheet_unresolved", "change_token_unavailable",
|
| 5574 |
"sheet_refresh_required", "schema_not_loaded", "grants_changed", "relation_cursor_mismatch",
|
|
|
|
| 5575 |
))
|
| 5576 |
|
| 5577 |
#: The binding itself is invalid for this reader: a field or database is gone, or access is denied.
|
|
@@ -6016,7 +6136,7 @@ def _resolve_binding(session: Session, raw_binding, *, today=None, view_memo=Non
|
|
| 6016 |
# and refuses 409 when one is missing, so it still runs AFTER the mirror exists, below.
|
| 6017 |
_refuse_unapplied_audience(session, database)
|
| 6018 |
wall_tree = ((perm_scope.entry(session.user, database) or {}).get("filter")
|
| 6019 |
-
if
|
| 6020 |
wall_nodes, wall_conj = _fe_parts(wall_tree)
|
| 6021 |
scope_nodes, scope_conj = _binding_filter_nodes(session, database, binding, memo=view_memo)
|
| 6022 |
scope_nodes = _filter_cell_values(session, scope_nodes, book=book, tab=tab,
|
|
|
|
| 207 |
return any(_database_catching_up(session, target) for target in targets)
|
| 208 |
|
| 209 |
|
| 210 |
+
#: Builds of one registry mirror per cell. A source write during a build costs a rebuild.
|
| 211 |
+
_MIRROR_BUILD_TRIES = 3
|
| 212 |
+
|
| 213 |
+
|
| 214 |
+
def _request_memo(session):
|
| 215 |
+
"""The Sheet request memo, or None outside a request scope or inside an isolated drill."""
|
| 216 |
+
memo = getattr(session, "_sheet_read_memo", None)
|
| 217 |
+
if not isinstance(memo, dict) or getattr(session, "_sheet_drill_isolated", False):
|
| 218 |
+
return None
|
| 219 |
+
return memo
|
| 220 |
+
|
| 221 |
+
|
| 222 |
+
def _request_mirror_cur(session):
|
| 223 |
+
"""One gated mirror cursor per Sheet request and mirror install generation."""
|
| 224 |
+
import routes_odoo_tables as rot
|
| 225 |
+
memo = _request_memo(session)
|
| 226 |
+
generation = _install_generation() if memo is not None else None
|
| 227 |
+
if generation is None:
|
| 228 |
+
return rot._mirror_cur(session)
|
| 229 |
+
hit = memo.get("mirror_cur")
|
| 230 |
+
if hit is not None and hit[0] == generation:
|
| 231 |
+
return hit[1]
|
| 232 |
+
got = rot._mirror_cur(session)
|
| 233 |
+
if got[1] is None:
|
| 234 |
+
memo["mirror_cur"] = (generation, got)
|
| 235 |
+
return got
|
| 236 |
+
|
| 237 |
+
|
| 238 |
+
def _request_source_for(session, cur, table_key):
|
| 239 |
+
"""`rot._source_for` once per Sheet request, cursor and database. Its catalog reads cost per call."""
|
| 240 |
+
import routes_odoo_tables as rot
|
| 241 |
+
memo = _request_memo(session)
|
| 242 |
+
if memo is None:
|
| 243 |
+
return rot._source_for(cur, table_key)
|
| 244 |
+
slot = ("source_for", table_key)
|
| 245 |
+
hit = memo.get(slot)
|
| 246 |
+
if hit is not None and hit[0] is cur:
|
| 247 |
+
return hit[1]
|
| 248 |
+
spec = rot._source_for(cur, table_key)
|
| 249 |
+
if spec is not None:
|
| 250 |
+
memo[slot] = (cur, spec)
|
| 251 |
+
return spec
|
| 252 |
+
|
| 253 |
+
|
| 254 |
+
def _row_scope_applies(session, database):
|
| 255 |
+
"""`perm_scope.row_scope_applies` once per Sheet request: its storeless form reads a revision."""
|
| 256 |
+
import core.perm_scope as perm_scope
|
| 257 |
+
memo = _request_memo(session)
|
| 258 |
+
slot = ("row_scope_applies", database)
|
| 259 |
+
if memo is not None and slot in memo:
|
| 260 |
+
return memo[slot]
|
| 261 |
+
applies = perm_scope.row_scope_applies(session.user, database)
|
| 262 |
+
if memo is not None:
|
| 263 |
+
memo[slot] = applies
|
| 264 |
+
return applies
|
| 265 |
+
|
| 266 |
+
|
| 267 |
+
def _registry_mirror_settled(session, cur, table_key, metric_window, needed):
|
| 268 |
+
"""Rebuild when the source changed during the build. Only the last change refuses."""
|
| 269 |
+
for attempt in range(_MIRROR_BUILD_TRIES):
|
| 270 |
+
try:
|
| 271 |
+
return _registry_mirror(session, cur, table_key, metric_window, needed=needed)
|
| 272 |
+
except HTTPException as exc:
|
| 273 |
+
detail = exc.detail if isinstance(exc.detail, dict) else {}
|
| 274 |
+
if ((detail.get("error") or {}).get("code") != "drill_population_changed"
|
| 275 |
+
or attempt + 1 >= _MIRROR_BUILD_TRIES):
|
| 276 |
+
raise
|
| 277 |
+
print(f"[sheet] mirror {table_key} changed during its build, rebuild {attempt + 2}",
|
| 278 |
+
flush=True)
|
| 279 |
+
|
| 280 |
+
|
| 281 |
+
def _hint_mirror_columns(session, book, only=None):
|
| 282 |
+
"""Name every column one refresh reads per registry database, so its mirror builds once."""
|
| 283 |
+
import odoo_relational as rel
|
| 284 |
+
from modules import sheet as sheet_mod
|
| 285 |
+
from core.request_timing import raise_if_resource_error
|
| 286 |
+
context = getattr(session, "_sheet_scope_context", None)
|
| 287 |
+
if not isinstance(context, dict):
|
| 288 |
+
return
|
| 289 |
+
hint = {}
|
| 290 |
+
for tab in book.get("sheets") or []:
|
| 291 |
+
for addr, cell in (tab.get("cells") or {}).items():
|
| 292 |
+
if (not isinstance(cell, dict) or cell.get("kind") != "bound" or _is_view_binding(cell)
|
| 293 |
+
or (only is not None and f"{tab.get('id')}!{addr}" not in only)):
|
| 294 |
+
continue
|
| 295 |
+
try:
|
| 296 |
+
binding = cell.get("binding") if isinstance(cell.get("binding"), dict) else {}
|
| 297 |
+
database = str(binding.get("database") or "")
|
| 298 |
+
scope = binding.get("scope") if isinstance(binding.get("scope"), dict) else {}
|
| 299 |
+
nodes, _conj = _fe_parts(scope.get("filter"))
|
| 300 |
+
related = sheet_mod.related_filter_ids(nodes)
|
| 301 |
+
if database not in rel._REGISTRY_GRID_FIELDS and not related:
|
| 302 |
+
continue
|
| 303 |
+
columns = hint.setdefault(database, set())
|
| 304 |
+
columns.update(name for name in sheet_mod.filter_columns_named(nodes) - related
|
| 305 |
+
if ":" not in str(name))
|
| 306 |
+
field = (binding.get("aggregate") or {}).get("field")
|
| 307 |
+
if field:
|
| 308 |
+
columns.add(str(field))
|
| 309 |
+
for key in sorted(related):
|
| 310 |
+
via, field = sheet_mod.related_field_parts(key)
|
| 311 |
+
entry = _binding_related_entry(session, database, via)
|
| 312 |
+
if entry["via"].get("from"):
|
| 313 |
+
columns.add(str(entry["via"]["from"]))
|
| 314 |
+
hint.setdefault(entry["target"], set()).update(
|
| 315 |
+
{str(field), str(entry["via"].get("on") or field)})
|
| 316 |
+
except Exception as exc: # noqa: BLE001
|
| 317 |
+
# The cell's own resolution reports its refusal. A hint only widens a build.
|
| 318 |
+
raise_if_resource_error(exc)
|
| 319 |
+
continue
|
| 320 |
+
context[("mirror_columns",)] = hint
|
| 321 |
+
|
| 322 |
+
|
| 323 |
def _mirror(session: Session):
|
| 324 |
"""`(cursor, spec, columns)` for one read-through database, or the refusal that names why.
|
| 325 |
|
|
|
|
| 391 |
session._sheet_amzn_cursor = (cur, None)
|
| 392 |
else:
|
| 393 |
cur, mismatch = (getattr(session, "_sheet_drill_cursor", None)
|
| 394 |
+
or _request_mirror_cur(session))
|
| 395 |
except RuntimeError as exc:
|
| 396 |
raise err(503, "store_not_ready", str(exc))
|
| 397 |
if mismatch:
|
|
|
|
| 463 |
session._sheet_mirrors = memo
|
| 464 |
slot = (table_key, json.dumps(metric_window, sort_keys=True, default=str))
|
| 465 |
want = None if needed is None else {str(k) for k in needed}
|
| 466 |
+
context = getattr(session, "_sheet_scope_context", None)
|
| 467 |
+
hint = (context.get(("mirror_columns",)) or {}).get(table_key) if isinstance(context, dict) else None
|
| 468 |
+
if want is not None and hint:
|
| 469 |
+
want |= hint
|
| 470 |
entry = memo.get(slot)
|
| 471 |
if entry is not None:
|
| 472 |
built, made = entry
|
|
|
|
| 474 |
return made
|
| 475 |
if want is not None:
|
| 476 |
want = set(want) | built
|
| 477 |
+
made = _registry_mirror_settled(session, cur, table_key, metric_window, want)
|
| 478 |
memo[slot] = (None if want is None else set(want), made)
|
| 479 |
return made
|
| 480 |
+
spec = _request_source_for(session, cur, table_key)
|
| 481 |
if spec is None and connector_defn is not None and str(table_key).startswith("ut_amzn_"):
|
| 482 |
spec = _amazon_stored_spec(cur, table_key)
|
| 483 |
if spec is None:
|
|
|
|
| 4566 |
return hit["cell"]
|
| 4567 |
return None
|
| 4568 |
|
| 4569 |
+
_hint_mirror_columns(session, book, only)
|
| 4570 |
started = perf_counter()
|
| 4571 |
resolved_count = 0
|
| 4572 |
held_refusal = None
|
|
|
|
| 5682 |
"store_not_ready": "#UPDATING",
|
| 5683 |
"mirror_not_ready": "#UPDATING",
|
| 5684 |
"amazon_mirror_not_loaded": "#UPDATING",
|
| 5685 |
+
"drill_population_changed": "#UPDATING",
|
| 5686 |
}
|
| 5687 |
|
| 5688 |
|
|
|
|
| 5691 |
"store_catching_up", "source_unavailable", "store_not_ready", "mirror_not_ready",
|
| 5692 |
"mirror_mismatch", "amazon_mirror_not_loaded", "sheet_unresolved", "change_token_unavailable",
|
| 5693 |
"sheet_refresh_required", "schema_not_loaded", "grants_changed", "relation_cursor_mismatch",
|
| 5694 |
+
"drill_population_changed",
|
| 5695 |
))
|
| 5696 |
|
| 5697 |
#: The binding itself is invalid for this reader: a field or database is gone, or access is denied.
|
|
|
|
| 6136 |
# and refuses 409 when one is missing, so it still runs AFTER the mirror exists, below.
|
| 6137 |
_refuse_unapplied_audience(session, database)
|
| 6138 |
wall_tree = ((perm_scope.entry(session.user, database) or {}).get("filter")
|
| 6139 |
+
if _row_scope_applies(session, database) else None)
|
| 6140 |
wall_nodes, wall_conj = _fe_parts(wall_tree)
|
| 6141 |
scope_nodes, scope_conj = _binding_filter_nodes(session, database, binding, memo=view_memo)
|
| 6142 |
scope_nodes = _filter_cell_values(session, scope_nodes, book=book, tab=tab,
|
api/routes_starred.py
CHANGED
|
@@ -329,7 +329,8 @@ def _scan_starred_views(session, keys=None, cache=None, include=None, with_right
|
|
| 329 |
import core.table_store as table_store
|
| 330 |
import core.view_templates as view_templates
|
| 331 |
|
| 332 |
-
|
|
|
|
| 333 |
# ⭐⭐ W62-T17 (D-756) — EVERY GRANTED VIEW ID WAS RESOLVED ONCE PER DATABASE, and that loop is
|
| 334 |
# where `/starred`'s budget went. MEASURED on staging's Royal Imports, 26 databases
|
| 335 |
# (`AutoWiki/waves/wave62/proto/T17-profile.md`): 4,844 ms total, of which `_visible_views` was
|
|
|
|
| 329 |
import core.table_store as table_store
|
| 330 |
import core.view_templates as view_templates
|
| 331 |
|
| 332 |
+
# The docs this scan holds resolve folder-shared Views without a second read per database.
|
| 333 |
+
granted = routes_nav._granted_view_sources(session, docs=docs)
|
| 334 |
# ⭐⭐ W62-T17 (D-756) — EVERY GRANTED VIEW ID WAS RESOLVED ONCE PER DATABASE, and that loop is
|
| 335 |
# where `/starred`'s budget went. MEASURED on staging's Royal Imports, 26 databases
|
| 336 |
# (`AutoWiki/waves/wave62/proto/T17-profile.md`): 4,844 ms total, of which `_visible_views` was
|
platform/core/audit.py
CHANGED
|
@@ -1,7 +1,9 @@
|
|
| 1 |
"""Bounded, tenant-scoped security events. Refuse secrets before attempting any store I/O."""
|
| 2 |
from collections.abc import Mapping
|
|
|
|
| 3 |
import math
|
| 4 |
import re
|
|
|
|
| 5 |
import time
|
| 6 |
|
| 7 |
from harness import runtime
|
|
@@ -21,6 +23,11 @@ BUCKET = "security_events"
|
|
| 21 |
MAX_EVENTS = 2000
|
| 22 |
EXTERNAL_KINDS = frozenset({"external.query", "external.write"})
|
| 23 |
RETENTION_SECONDS = 400 * 24 * 60 * 60
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 24 |
_ATTRIBUTION_MAX = 256
|
| 25 |
_KEY_MAX = 128
|
| 26 |
|
|
@@ -161,37 +168,131 @@ def ordered(doc, tenant, limit=200, *, kinds=None) -> list[dict]:
|
|
| 161 |
return sorted(retained, key=lambda row: row["ts"])[-limit:]
|
| 162 |
|
| 163 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 164 |
def record(kind, tenant, principal, detail=None) -> dict | None:
|
| 165 |
"""Refusals raise before I/O. Store failures lose the event, never the user's sign-in."""
|
| 166 |
item = event(kind, tenant, principal, detail)
|
| 167 |
if freeze_notice.enabled():
|
| 168 |
return None
|
| 169 |
tenant = item["tenant"]
|
| 170 |
-
|
| 171 |
-
|
| 172 |
-
|
| 173 |
-
|
| 174 |
-
|
| 175 |
-
rows = [row for row in rows if row["ts"] >= item["ts"] - RETENTION_SECONDS]
|
| 176 |
-
rows.sort(key=lambda row: row["ts"])
|
| 177 |
-
# Sign-ins and external calls each own their 2000-event retention partition.
|
| 178 |
-
external = [row for row in rows if row["kind"] in EXTERNAL_KINDS][-MAX_EVENTS:]
|
| 179 |
-
security = [row for row in rows if row["kind"] not in EXTERNAL_KINDS][-MAX_EVENTS:]
|
| 180 |
-
return {"events": sorted(external + security, key=lambda row: row["ts"])}
|
| 181 |
-
|
| 182 |
try:
|
| 183 |
-
|
| 184 |
-
# ⭐⭐ flush="async" IS DELIBERATE AND IT IS NOT A PERFORMANCE TWEAK. The default
|
| 185 |
-
# flush="sync" is a BLOCKING upload (core/store.py::update says so), and this writer sits
|
| 186 |
-
# on the sign-in path: every login would then wait on a network round trip, and on the hf
|
| 187 |
-
# backend every login would spend one of that repo's 256 commits per hour. The async
|
| 188 |
-
# branch applies the mutation to the in-process cache and a coalescing worker commits it,
|
| 189 |
-
# which is the same posture the table-workspace hot path has used since wave 7.
|
| 190 |
-
# ⚠ THE COST, stated rather than hidden: a container that dies before the worker flushes
|
| 191 |
-
# loses the events still in its cache. That is the same window `record` already accepts by
|
| 192 |
-
# swallowing a store failure, and it is the right side of the trade: an audit log must
|
| 193 |
-
# never be the reason a person cannot sign in.
|
| 194 |
-
runtime.get_runtime(str(tenant)).update(BUCKET, _append, flush="async")
|
| 195 |
except Exception as exc:
|
| 196 |
# Exception text can contain credentials from a provider or connection string.
|
| 197 |
print(f"[aios-api] audit lost {item['kind']} for tenant {tenant!r}: "
|
|
@@ -204,6 +305,8 @@ def read(tenant, limit=200, now=None, *, kinds=None, viewer=None) -> list[dict]:
|
|
| 204 |
"""Read one tenant's retained events. Read failures propagate instead of claiming no history."""
|
| 205 |
tenant = _attribution("tenant", tenant)
|
| 206 |
now = time.time() if now is None else float(now)
|
|
|
|
|
|
|
| 207 |
doc = runtime.get_runtime(str(tenant)).get(BUCKET)
|
| 208 |
rows = ordered(doc, tenant, limit, kinds=kinds)
|
| 209 |
rows = [row for row in rows if row["ts"] >= now - RETENTION_SECONDS]
|
|
|
|
| 1 |
"""Bounded, tenant-scoped security events. Refuse secrets before attempting any store I/O."""
|
| 2 |
from collections.abc import Mapping
|
| 3 |
+
import atexit
|
| 4 |
import math
|
| 5 |
import re
|
| 6 |
+
import threading
|
| 7 |
import time
|
| 8 |
|
| 9 |
from harness import runtime
|
|
|
|
| 23 |
MAX_EVENTS = 2000
|
| 24 |
EXTERNAL_KINDS = frozenset({"external.query", "external.write"})
|
| 25 |
RETENTION_SECONDS = 400 * 24 * 60 * 60
|
| 26 |
+
#: Kinds a write-through store queues and writes in batches, so one query never pays a document write.
|
| 27 |
+
DEFERRED_KINDS = frozenset({"external.query"})
|
| 28 |
+
#: A tenant queue this long is written by the caller that fills it: the queue never grows past it.
|
| 29 |
+
QUEUE_MAX = 200
|
| 30 |
+
FLUSH_INTERVAL_S = 2.0
|
| 31 |
_ATTRIBUTION_MAX = 256
|
| 32 |
_KEY_MAX = 128
|
| 33 |
|
|
|
|
| 168 |
return sorted(retained, key=lambda row: row["ts"])[-limit:]
|
| 169 |
|
| 170 |
|
| 171 |
+
def _well_formed(row, tenant):
|
| 172 |
+
"""A stored row this writer may keep without revalidation. Readers still revalidate every row."""
|
| 173 |
+
return (isinstance(row, dict) and row.get("tenant") == tenant
|
| 174 |
+
and type(row.get("ts")) in (int, float) and row.get("kind") in AUDIT_KINDS
|
| 175 |
+
and isinstance(row.get("principal"), str) and isinstance(row.get("detail"), dict))
|
| 176 |
+
|
| 177 |
+
|
| 178 |
+
def _append_rows(doc, tenant, items):
|
| 179 |
+
"""`{"events": rows}` with `items` added, pruned by age and capped per partition.
|
| 180 |
+
|
| 181 |
+
The common case is linear with no per-row validation. A document holding any row this tenant's
|
| 182 |
+
writer did not produce takes the full `ordered()` rebuild, which drops it.
|
| 183 |
+
"""
|
| 184 |
+
stored = doc.get("events") if isinstance(doc, Mapping) else None
|
| 185 |
+
stored = stored if isinstance(stored, list) else []
|
| 186 |
+
if all(_well_formed(row, tenant) for row in stored):
|
| 187 |
+
rows = list(stored)
|
| 188 |
+
else:
|
| 189 |
+
rows = ordered(doc, tenant, MAX_EVENTS * 2)
|
| 190 |
+
rows.extend(items)
|
| 191 |
+
# A queued batch can land after a later sign-in, so restore ascending order when it did.
|
| 192 |
+
if any(rows[i]["ts"] > rows[i + 1]["ts"] for i in range(len(rows) - 1)):
|
| 193 |
+
rows.sort(key=lambda row: row["ts"])
|
| 194 |
+
cutoff = max(item["ts"] for item in items) - RETENTION_SECONDS
|
| 195 |
+
if rows and rows[0]["ts"] < cutoff:
|
| 196 |
+
rows = [row for row in rows if row["ts"] >= cutoff]
|
| 197 |
+
# Sign-ins and external calls each own their 2000-event retention partition, oldest out first.
|
| 198 |
+
external = sum(1 for row in rows if row["kind"] in EXTERNAL_KINDS)
|
| 199 |
+
drop = {True: max(0, external - MAX_EVENTS), False: max(0, len(rows) - external - MAX_EVENTS)}
|
| 200 |
+
if drop[True] or drop[False]:
|
| 201 |
+
kept = []
|
| 202 |
+
for row in rows:
|
| 203 |
+
side = row["kind"] in EXTERNAL_KINDS
|
| 204 |
+
if drop[side]:
|
| 205 |
+
drop[side] -= 1
|
| 206 |
+
continue
|
| 207 |
+
kept.append(row)
|
| 208 |
+
rows = kept
|
| 209 |
+
return {"events": rows}
|
| 210 |
+
|
| 211 |
+
|
| 212 |
+
def _write(tenant, items):
|
| 213 |
+
# Bare core.store.update is bound to tenant #0. Always use the tenant-scoped door.
|
| 214 |
+
# ⭐⭐ flush="async" IS DELIBERATE AND IT IS NOT A PERFORMANCE TWEAK. The default
|
| 215 |
+
# flush="sync" is a BLOCKING upload (core/store.py::update says so), and this writer sits
|
| 216 |
+
# on the sign-in path: every login would then wait on a network round trip, and on the hf
|
| 217 |
+
# backend every login would spend one of that repo's 256 commits per hour. The async
|
| 218 |
+
# branch applies the mutation to the in-process cache and a coalescing worker commits it,
|
| 219 |
+
# which is the same posture the table-workspace hot path has used since wave 7.
|
| 220 |
+
# ⚠ THE COST, stated rather than hidden: a container that dies before the worker flushes
|
| 221 |
+
# loses the events still in its cache. That is the same window `record` already accepts by
|
| 222 |
+
# swallowing a store failure, and it is the right side of the trade: an audit log must
|
| 223 |
+
# never be the reason a person cannot sign in.
|
| 224 |
+
runtime.get_runtime(str(tenant)).update(
|
| 225 |
+
BUCKET, lambda doc: _append_rows(doc, tenant, items), flush="async")
|
| 226 |
+
|
| 227 |
+
|
| 228 |
+
def _writes_through(tenant):
|
| 229 |
+
"""True when the tenant store ignores flush="async" and commits the whole document per call (Postgres)."""
|
| 230 |
+
try:
|
| 231 |
+
backend = runtime.get_runtime(str(tenant))._document_store()
|
| 232 |
+
except Exception:
|
| 233 |
+
return False
|
| 234 |
+
return type(backend).__module__ == "core.store_pg"
|
| 235 |
+
|
| 236 |
+
|
| 237 |
+
_QUEUE = {}
|
| 238 |
+
_QUEUE_LOCK = threading.Lock()
|
| 239 |
+
_WAKE = threading.Event()
|
| 240 |
+
_FLUSHER = []
|
| 241 |
+
|
| 242 |
+
|
| 243 |
+
def _flush_loop():
|
| 244 |
+
while True:
|
| 245 |
+
_WAKE.wait(FLUSH_INTERVAL_S)
|
| 246 |
+
_WAKE.clear()
|
| 247 |
+
try:
|
| 248 |
+
flush_pending()
|
| 249 |
+
except Exception:
|
| 250 |
+
pass
|
| 251 |
+
|
| 252 |
+
|
| 253 |
+
def _enqueue(tenant, item):
|
| 254 |
+
with _QUEUE_LOCK:
|
| 255 |
+
pending = _QUEUE.setdefault(tenant, [])
|
| 256 |
+
pending.append(item)
|
| 257 |
+
full = len(pending) >= QUEUE_MAX
|
| 258 |
+
if not _FLUSHER:
|
| 259 |
+
_FLUSHER.append(threading.Thread(target=_flush_loop, name="audit-flush", daemon=True))
|
| 260 |
+
_FLUSHER[0].start()
|
| 261 |
+
atexit.register(flush_pending)
|
| 262 |
+
if full:
|
| 263 |
+
flush_pending(tenant)
|
| 264 |
+
|
| 265 |
+
|
| 266 |
+
def flush_pending(tenant=None) -> int:
|
| 267 |
+
"""Write queued events in one update per tenant. Returns how many rows were handed to the store."""
|
| 268 |
+
with _QUEUE_LOCK:
|
| 269 |
+
names = list(_QUEUE) if tenant is None else [_attribution("tenant", tenant)]
|
| 270 |
+
batches = {name: _QUEUE.pop(name) for name in names if _QUEUE.get(name)}
|
| 271 |
+
written = 0
|
| 272 |
+
for name, items in batches.items():
|
| 273 |
+
try:
|
| 274 |
+
_write(name, items)
|
| 275 |
+
written += len(items)
|
| 276 |
+
except Exception as exc:
|
| 277 |
+
# Exception text can contain credentials from a provider or connection string.
|
| 278 |
+
print(f"[aios-api] audit lost {len(items)} queued event(s) for tenant {name!r}: "
|
| 279 |
+
f"store unavailable ({type(exc).__name__}).")
|
| 280 |
+
return written
|
| 281 |
+
|
| 282 |
+
|
| 283 |
def record(kind, tenant, principal, detail=None) -> dict | None:
|
| 284 |
"""Refusals raise before I/O. Store failures lose the event, never the user's sign-in."""
|
| 285 |
item = event(kind, tenant, principal, detail)
|
| 286 |
if freeze_notice.enabled():
|
| 287 |
return None
|
| 288 |
tenant = item["tenant"]
|
| 289 |
+
# Sign-in and security kinds always write now. A query event on a write-through store is
|
| 290 |
+
# queued, because that store would otherwise rewrite the whole log on the request path.
|
| 291 |
+
if kind in DEFERRED_KINDS and _writes_through(tenant):
|
| 292 |
+
_enqueue(tenant, item)
|
| 293 |
+
return item
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 294 |
try:
|
| 295 |
+
_write(tenant, [item])
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 296 |
except Exception as exc:
|
| 297 |
# Exception text can contain credentials from a provider or connection string.
|
| 298 |
print(f"[aios-api] audit lost {item['kind']} for tenant {tenant!r}: "
|
|
|
|
| 305 |
"""Read one tenant's retained events. Read failures propagate instead of claiming no history."""
|
| 306 |
tenant = _attribution("tenant", tenant)
|
| 307 |
now = time.time() if now is None else float(now)
|
| 308 |
+
# Queued events are written first, so a reader sees every call this process accepted.
|
| 309 |
+
flush_pending(tenant)
|
| 310 |
doc = runtime.get_runtime(str(tenant)).get(BUCKET)
|
| 311 |
rows = ordered(doc, tenant, limit, kinds=kinds)
|
| 312 |
rows = [row for row in rows if row["ts"] >= now - RETENTION_SECONDS]
|
platform/harness/amazon_schema.py
CHANGED
|
@@ -411,7 +411,8 @@ TABLE_SPECS = {
|
|
| 411 |
("msku", "listings", "seller_sku", "many_to_one"),
|
| 412 |
("asin", "listings", "asin1", "many_to_one"),
|
| 413 |
)),
|
| 414 |
-
|
|
|
|
| 415 |
("child_asin", "listings", "asin1", "many_to_one"),
|
| 416 |
("sku", "listings", "seller_sku", "many_to_one"),
|
| 417 |
)),
|
|
|
|
| 411 |
("msku", "listings", "seller_sku", "many_to_one"),
|
| 412 |
("asin", "listings", "asin1", "many_to_one"),
|
| 413 |
)),
|
| 414 |
+
# The period, not the parent, is identity: a re-parented child replaces its own window's row.
|
| 415 |
+
"pricing": _table_spec("pricing", ("child_asin", "sku", "_period_start", "_period_end"), (
|
| 416 |
("child_asin", "listings", "asin1", "many_to_one"),
|
| 417 |
("sku", "listings", "seller_sku", "many_to_one"),
|
| 418 |
)),
|
platform/harness/amazon_store.py
CHANGED
|
@@ -253,6 +253,8 @@ FIRST_LOAD_REPORTS_PER_PASS = 15
|
|
| 253 |
#: minutes each) never hold the sync desk for hours (D-7081).
|
| 254 |
FIRST_LOAD_ROWS_PER_PASS = 3_000_000
|
| 255 |
BACKFILL_FLOOR_PREFIX = "sync_floor_"
|
|
|
|
|
|
|
| 256 |
FIRST_LOAD_END_KEY = "sync_first_load_end"
|
| 257 |
FIRST_LOAD_STATUS_KEY = "sync_first_load_status"
|
| 258 |
FIRST_LOAD_DUE_KEY = "sync_first_load_next_due"
|
|
@@ -2984,7 +2986,7 @@ class _AmazonMirrorStore(_mirrors.MirrorStore):
|
|
| 2984 |
def owner(entity):
|
| 2985 |
if entity.startswith(entity_key("settlement_doc_")):
|
| 2986 |
return table_name("settlements")
|
| 2987 |
-
for family in (WATERMARK_PREFIX, BACKFILL_FLOOR_PREFIX, "reconcile_", "limit_"):
|
| 2988 |
for name in names:
|
| 2989 |
marker = entity_key(family + name[len(PREFIX):])
|
| 2990 |
if entity == marker or entity.startswith(marker + "_"):
|
|
@@ -3003,8 +3005,15 @@ class _AmazonMirrorStore(_mirrors.MirrorStore):
|
|
| 3003 |
def _after_restore(self, con, result, log):
|
| 3004 |
_heal_snapshots(con, result, log)
|
| 3005 |
_heal_quoted_values(con, result, log)
|
|
|
|
| 3006 |
refresh_sku_keywords(con)
|
| 3007 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3008 |
def _restored_state_rows(self, rows, tables, refused):
|
| 3009 |
return _restored_state_rows(rows, tables, refused)
|
| 3010 |
|
|
@@ -3062,7 +3071,8 @@ def _restored_state_rows(rows, tables, refused):
|
|
| 3062 |
return rows
|
| 3063 |
refused = set(refused)
|
| 3064 |
names = sorted(set(tables) | {table_name(t) for t in TABLES}, key=lambda name: (-len(name), name))
|
| 3065 |
-
families = ("", WATERMARK_PREFIX, BACKFILL_FLOOR_PREFIX, "sync_tick_", "reconcile_", "limit_", "refused_"
|
|
|
|
| 3066 |
shared = {entity_key(key) for key in (FIRST_LOAD_END_KEY, FIRST_LOAD_STATUS_KEY, FIRST_LOAD_DUE_KEY)}
|
| 3067 |
|
| 3068 |
def keep(row):
|
|
@@ -3208,6 +3218,7 @@ def _unquote_mirror_values(con, log=None):
|
|
| 3208 |
stale = " OR ".join(f"{_quoted_identifier(c)} LIKE '\"%\"'" for c in identity)
|
| 3209 |
if not con.execute(f"SELECT 1 FROM {table} WHERE {stale} LIMIT 1").fetchone():
|
| 3210 |
continue
|
|
|
|
| 3211 |
sets = []
|
| 3212 |
dated = mirror[len(PREFIX):] in APP_DATE_TABLES
|
| 3213 |
for col in text:
|
|
@@ -3220,6 +3231,7 @@ def _unquote_mirror_values(con, log=None):
|
|
| 3220 |
con.execute("BEGIN TRANSACTION")
|
| 3221 |
try:
|
| 3222 |
healed[mirror] = con.execute(f"SELECT count(*) FROM {table} WHERE {stale}").fetchone()[0]
|
|
|
|
| 3223 |
con.execute(f"UPDATE {table} SET {', '.join(sets)} WHERE {stale}")
|
| 3224 |
con.execute("COMMIT")
|
| 3225 |
except Exception:
|
|
@@ -3228,9 +3240,139 @@ def _unquote_mirror_values(con, log=None):
|
|
| 3228 |
raise
|
| 3229 |
if log:
|
| 3230 |
log(f"Amazon quoted values {mirror}: {healed[mirror]:,} rows unquoted")
|
|
|
|
|
|
|
|
|
|
| 3231 |
return healed
|
| 3232 |
|
| 3233 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3234 |
def _heal_quoted_values(con, result, log):
|
| 3235 |
"""Unquote after a restore; healed tables are dirty so the next checkpoint saves them clean."""
|
| 3236 |
try:
|
|
@@ -3595,7 +3737,8 @@ def _record_state(con, key, phase, rows=0, detail=None):
|
|
| 3595 |
con.execute(
|
| 3596 |
"INSERT OR REPLACE INTO _sync_state "
|
| 3597 |
"(entity, phase, last_id, cursor_wd, rows, updated_at) VALUES (?,?,?,?,?,?)",
|
| 3598 |
-
[str(key), str(phase), None, (str(detail) if str(key).startswith(("amzn_reconcile_", "amzn_limit_", "amzn_refused_", "amzn_pending_report_"
|
|
|
|
| 3599 |
else str(detail)[:300]) if detail else None, int(rows or 0),
|
| 3600 |
_iso(datetime.now(timezone.utc))])
|
| 3601 |
|
|
@@ -5202,12 +5345,17 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
|
|
| 5202 |
except Exception as exc: # noqa: BLE001
|
| 5203 |
report["problems"].append(f"Amazon snapshot collapse failed ({type(exc).__name__}); the next "
|
| 5204 |
"resync of each snapshot table replaces its earlier copies.")
|
| 5205 |
-
#
|
| 5206 |
try:
|
| 5207 |
changed_tables.update(_unquote_mirror_values(con, log))
|
| 5208 |
except Exception as exc: # noqa: BLE001
|
| 5209 |
report["problems"].append(f"Amazon quoted value heal failed ({type(exc).__name__}); the next "
|
| 5210 |
"resync of each quoted table writes clean values.")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5211 |
|
| 5212 |
def write_window(table, cols, types, rows):
|
| 5213 |
if cx.REPORT_TYPES[table].get("weekly"):
|
|
@@ -5477,7 +5625,14 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
|
|
| 5477 |
margin_windows = _month_windows(plan["start"], plan["end"])
|
| 5478 |
else:
|
| 5479 |
margin_windows = _windows(plan["start"], plan["end"], spec.get("max_window_days", MAX_WINDOW_DAYS))
|
| 5480 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5481 |
# One due window per table tick respects the vendor floor. The watermark
|
| 5482 |
# advances only to that window's end, so the following tick loses no history.
|
| 5483 |
remaining_windows = len(margin_windows) - 1
|
|
@@ -5500,6 +5655,8 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
|
|
| 5500 |
{"state": "access", "operation": operation}, rt, report, log)
|
| 5501 |
count = _sync_margin_window(con, table, fields, w0, w1,
|
| 5502 |
max_wait=max_wait, log=log)
|
|
|
|
|
|
|
| 5503 |
probe.finish(count)
|
| 5504 |
spend_rows(count)
|
| 5505 |
written += count
|
|
@@ -6927,6 +7084,91 @@ def mirror_selftest(log=print, check=None):
|
|
| 6927 |
whole["state"] == "refused" and whole["refused_tables"] == []
|
| 6928 |
and dump() == {"_sync_state": []}
|
| 6929 |
and not any(path.endswith(".parquet") for path in downloaded))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6930 |
finally:
|
| 6931 |
forget()
|
| 6932 |
con.close()
|
|
@@ -7725,6 +7967,23 @@ def group_a_selftest(log=print, check=None):
|
|
| 7725 |
con.execute("DELETE FROM _sync_state WHERE entity=?", [entity_key(refused_table)])
|
| 7726 |
ck("NC: the same table never refused (no watermark) still owes a first load",
|
| 7727 |
needs_first_load(rt.key, rt))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 7728 |
finally:
|
| 7729 |
con.close()
|
| 7730 |
|
|
|
|
| 253 |
#: minutes each) never hold the sync desk for hours (D-7081).
|
| 254 |
FIRST_LOAD_ROWS_PER_PASS = 3_000_000
|
| 255 |
BACKFILL_FLOOR_PREFIX = "sync_floor_"
|
| 256 |
+
#: Report windows whose stored rows cannot be healed in place; the next pass downloads them again.
|
| 257 |
+
RESYNC_PREFIX = "resync_"
|
| 258 |
FIRST_LOAD_END_KEY = "sync_first_load_end"
|
| 259 |
FIRST_LOAD_STATUS_KEY = "sync_first_load_status"
|
| 260 |
FIRST_LOAD_DUE_KEY = "sync_first_load_next_due"
|
|
|
|
| 2986 |
def owner(entity):
|
| 2987 |
if entity.startswith(entity_key("settlement_doc_")):
|
| 2988 |
return table_name("settlements")
|
| 2989 |
+
for family in (WATERMARK_PREFIX, BACKFILL_FLOOR_PREFIX, "reconcile_", "limit_", RESYNC_PREFIX):
|
| 2990 |
for name in names:
|
| 2991 |
marker = entity_key(family + name[len(PREFIX):])
|
| 2992 |
if entity == marker or entity.startswith(marker + "_"):
|
|
|
|
| 3005 |
def _after_restore(self, con, result, log):
|
| 3006 |
_heal_snapshots(con, result, log)
|
| 3007 |
_heal_quoted_values(con, result, log)
|
| 3008 |
+
_heal_identity_duplicates(con, result, log)
|
| 3009 |
refresh_sku_keywords(con)
|
| 3010 |
|
| 3011 |
+
def _heal_restored(self, con, result, log):
|
| 3012 |
+
# Heal the served rows without marking them dirty: a dirty set makes the retention sweep publish.
|
| 3013 |
+
healed = {"dirty": set(), "problems": result["problems"]}
|
| 3014 |
+
for heal in (_heal_snapshots, _heal_quoted_values, _heal_identity_duplicates):
|
| 3015 |
+
heal(con, healed, log)
|
| 3016 |
+
|
| 3017 |
def _restored_state_rows(self, rows, tables, refused):
|
| 3018 |
return _restored_state_rows(rows, tables, refused)
|
| 3019 |
|
|
|
|
| 3071 |
return rows
|
| 3072 |
refused = set(refused)
|
| 3073 |
names = sorted(set(tables) | {table_name(t) for t in TABLES}, key=lambda name: (-len(name), name))
|
| 3074 |
+
families = ("", WATERMARK_PREFIX, BACKFILL_FLOOR_PREFIX, "sync_tick_", "reconcile_", "limit_", "refused_",
|
| 3075 |
+
RESYNC_PREFIX)
|
| 3076 |
shared = {entity_key(key) for key in (FIRST_LOAD_END_KEY, FIRST_LOAD_STATUS_KEY, FIRST_LOAD_DUE_KEY)}
|
| 3077 |
|
| 3078 |
def keep(row):
|
|
|
|
| 3218 |
stale = " OR ".join(f"{_quoted_identifier(c)} LIKE '\"%\"'" for c in identity)
|
| 3219 |
if not con.execute(f"SELECT 1 FROM {table} WHERE {stale} LIMIT 1").fetchone():
|
| 3220 |
continue
|
| 3221 |
+
lost = _unhealable_sql(types, text)
|
| 3222 |
sets = []
|
| 3223 |
dated = mirror[len(PREFIX):] in APP_DATE_TABLES
|
| 3224 |
for col in text:
|
|
|
|
| 3231 |
con.execute("BEGIN TRANSACTION")
|
| 3232 |
try:
|
| 3233 |
healed[mirror] = con.execute(f"SELECT count(*) FROM {table} WHERE {stale}").fetchone()[0]
|
| 3234 |
+
resync = _queue_resync(con, mirror, types, f"({stale}) AND ({lost})") if lost else None
|
| 3235 |
con.execute(f"UPDATE {table} SET {', '.join(sets)} WHERE {stale}")
|
| 3236 |
con.execute("COMMIT")
|
| 3237 |
except Exception:
|
|
|
|
| 3240 |
raise
|
| 3241 |
if log:
|
| 3242 |
log(f"Amazon quoted values {mirror}: {healed[mirror]:,} rows unquoted")
|
| 3243 |
+
if resync:
|
| 3244 |
+
log(f"Amazon resync {mirror}: {resync[0]:,} rows lost cells to the old parse; "
|
| 3245 |
+
f"{len(resync[1])} report windows from {resync[1][0][0]} download again on the next passes")
|
| 3246 |
return healed
|
| 3247 |
|
| 3248 |
|
| 3249 |
+
def _unhealable_sql(types, text):
|
| 3250 |
+
"""SQL true for a quoted row the unquote cannot repair, or "" when no such test applies.
|
| 3251 |
+
|
| 3252 |
+
A cell opening an unclosed quote is a value the old parse split at a tab, so every later cell
|
| 3253 |
+
shifted. A row whose every numeric measure is NULL lost them to the failed numeric cast.
|
| 3254 |
+
"""
|
| 3255 |
+
split = [f"({q} LIKE '\"%' AND length(regexp_extract(substr({q}, 2), '\"*$')) % 2 = 0)"
|
| 3256 |
+
for q in map(_quoted_identifier, text)]
|
| 3257 |
+
measures = [_quoted_identifier(c) for c, t in types.items()
|
| 3258 |
+
if str(t).upper() != "VARCHAR" and not str(c).startswith("_")]
|
| 3259 |
+
if measures:
|
| 3260 |
+
split.append("(" + " AND ".join(f"{q} IS NULL" for q in measures) + ")")
|
| 3261 |
+
return " OR ".join(split)
|
| 3262 |
+
|
| 3263 |
+
|
| 3264 |
+
def _queue_resync(con, mirror, types, where):
|
| 3265 |
+
"""Record the report windows of rows matching `where` for download again -> (rows, windows) or None.
|
| 3266 |
+
|
| 3267 |
+
Only a dated window table qualifies: its next pass replaces each queued `_period_start` whole.
|
| 3268 |
+
"""
|
| 3269 |
+
table = mirror[len(PREFIX):]
|
| 3270 |
+
if (table not in PERIOD_TABLES or not {"_period_start", "_period_end"} <= set(types)
|
| 3271 |
+
or (_client().REPORT_TYPES.get(table) or {}).get("snapshot")):
|
| 3272 |
+
return None
|
| 3273 |
+
found = con.execute(
|
| 3274 |
+
f"SELECT CAST(_period_start AS VARCHAR), CAST(_period_end AS VARCHAR), count(*) "
|
| 3275 |
+
f"FROM {_quoted_identifier(mirror)} WHERE ({where}) AND _period_start IS NOT NULL "
|
| 3276 |
+
f"AND _period_end IS NOT NULL GROUP BY 1, 2").fetchall()
|
| 3277 |
+
if not found:
|
| 3278 |
+
return None
|
| 3279 |
+
key = entity_key(RESYNC_PREFIX + table)
|
| 3280 |
+
saved = _read_state(con, key)
|
| 3281 |
+
try:
|
| 3282 |
+
queued = {tuple(w) for w in json.loads(saved["detail"])["windows"]} if saved else set()
|
| 3283 |
+
except (ValueError, KeyError, TypeError):
|
| 3284 |
+
queued = set()
|
| 3285 |
+
windows = sorted(queued | {(start, end) for start, end, _n in found})
|
| 3286 |
+
rows = sum(n for _s, _e, n in found)
|
| 3287 |
+
_record_state(con, key, "pending", rows, json.dumps({"windows": [list(w) for w in windows]}))
|
| 3288 |
+
return rows, windows
|
| 3289 |
+
|
| 3290 |
+
|
| 3291 |
+
def queued_resync(con, table):
|
| 3292 |
+
"""The oldest queued resync window of `table` as `(start, end)` datetimes, or None."""
|
| 3293 |
+
saved = _read_state(con, entity_key(RESYNC_PREFIX + table))
|
| 3294 |
+
try:
|
| 3295 |
+
start, end = json.loads(saved["detail"])["windows"][0]
|
| 3296 |
+
return _utc(start), _utc(end)
|
| 3297 |
+
except (TypeError, ValueError, KeyError, IndexError, AttributeError):
|
| 3298 |
+
return None
|
| 3299 |
+
|
| 3300 |
+
|
| 3301 |
+
def resync_landed(con, table, window):
|
| 3302 |
+
"""Drop one landed window from the queue; the marker goes when the queue is empty."""
|
| 3303 |
+
key = entity_key(RESYNC_PREFIX + table)
|
| 3304 |
+
saved = _read_state(con, key)
|
| 3305 |
+
try:
|
| 3306 |
+
windows = [w for w in json.loads(saved["detail"])["windows"]
|
| 3307 |
+
if (_utc(w[0]), _utc(w[1])) != tuple(window)]
|
| 3308 |
+
except (TypeError, ValueError, KeyError, AttributeError):
|
| 3309 |
+
windows = []
|
| 3310 |
+
if windows:
|
| 3311 |
+
_record_state(con, key, "pending", saved["rows"], json.dumps({"windows": windows}))
|
| 3312 |
+
else:
|
| 3313 |
+
con.execute("DELETE FROM _sync_state WHERE entity=?", [key])
|
| 3314 |
+
|
| 3315 |
+
|
| 3316 |
+
#: Tables whose declared identity can hold rows from two documents after an identity change.
|
| 3317 |
+
_IDENTITY_COLLAPSE_TABLES = ("pricing",)
|
| 3318 |
+
|
| 3319 |
+
|
| 3320 |
+
def _collapse_identity(con, log=None):
|
| 3321 |
+
"""Rows sharing a declared identity keep the newest landed document only -> {mirror: (before, after)}.
|
| 3322 |
+
|
| 3323 |
+
Newest is the document holding the highest `rowid` (landing order, the ranking diagnostic's
|
| 3324 |
+
rule), then the larger document ID. A row with no `_period_start` goes once the same child and
|
| 3325 |
+
SKU hold a dated row. Idempotent: one probe per table when nothing is doubled.
|
| 3326 |
+
"""
|
| 3327 |
+
existing = {name for name, kind in _amazon_objects(con) if kind == "BASE TABLE"}
|
| 3328 |
+
collapsed = {}
|
| 3329 |
+
for table in _IDENTITY_COLLAPSE_TABLES:
|
| 3330 |
+
mirror = table_name(table)
|
| 3331 |
+
if mirror not in existing:
|
| 3332 |
+
continue
|
| 3333 |
+
cols = _columns_of(con, mirror)
|
| 3334 |
+
keys = [c for c in replace_keys_for(table) if c in cols]
|
| 3335 |
+
if not keys or "_report_document_id" not in cols:
|
| 3336 |
+
continue
|
| 3337 |
+
q, target = _quoted_identifier, _quoted_identifier(mirror)
|
| 3338 |
+
part = ", ".join(q(c) for c in keys)
|
| 3339 |
+
doomed = (f"SELECT rid FROM (SELECT rid, d, first_value(d) OVER (PARTITION BY {part} "
|
| 3340 |
+
f"ORDER BY landed DESC, d DESC) AS keep FROM (SELECT rowid AS rid, {part}, "
|
| 3341 |
+
f"_report_document_id AS d, max(rowid) OVER (PARTITION BY _report_document_id) AS landed "
|
| 3342 |
+
f"FROM {target})) WHERE d IS DISTINCT FROM keep")
|
| 3343 |
+
if "_period_start" in keys:
|
| 3344 |
+
same = " AND ".join(f"o.{q(c)} IS NOT DISTINCT FROM t.{q(c)}" for c in keys
|
| 3345 |
+
if c not in ("_period_start", "_period_end")) or "TRUE"
|
| 3346 |
+
doomed += (f" UNION SELECT t.rowid FROM {target} AS t WHERE t._period_start IS NULL AND EXISTS "
|
| 3347 |
+
f"(SELECT 1 FROM {target} AS o WHERE o._period_start IS NOT NULL AND {same})")
|
| 3348 |
+
if not con.execute(f"SELECT 1 FROM ({doomed}) LIMIT 1").fetchone():
|
| 3349 |
+
continue
|
| 3350 |
+
before = con.execute(f"SELECT count(*) FROM {target}").fetchone()[0]
|
| 3351 |
+
con.execute("BEGIN TRANSACTION")
|
| 3352 |
+
try:
|
| 3353 |
+
con.execute(f"DELETE FROM {target} WHERE rowid IN ({doomed})")
|
| 3354 |
+
con.execute("COMMIT")
|
| 3355 |
+
except Exception:
|
| 3356 |
+
con.execute("ROLLBACK")
|
| 3357 |
+
raise
|
| 3358 |
+
after = con.execute(f"SELECT count(*) FROM {target}").fetchone()[0]
|
| 3359 |
+
collapsed[mirror] = (before, after)
|
| 3360 |
+
if log:
|
| 3361 |
+
log(f"Amazon identity {mirror}: {before:,} rows to {after:,}, one document per identity")
|
| 3362 |
+
return collapsed
|
| 3363 |
+
|
| 3364 |
+
|
| 3365 |
+
def _heal_identity_duplicates(con, result, log):
|
| 3366 |
+
"""Collapse doubled identities after a restore; healed tables are dirty for the next checkpoint."""
|
| 3367 |
+
try:
|
| 3368 |
+
result["dirty"].update(_collapse_identity(con, log))
|
| 3369 |
+
except Exception as exc: # noqa: BLE001
|
| 3370 |
+
message = (f"Amazon identity collapse failed ({type(exc).__name__}); the next resync of each "
|
| 3371 |
+
"doubled table replaces its earlier document's rows.")
|
| 3372 |
+
result["problems"].append(message)
|
| 3373 |
+
log(message)
|
| 3374 |
+
|
| 3375 |
+
|
| 3376 |
def _heal_quoted_values(con, result, log):
|
| 3377 |
"""Unquote after a restore; healed tables are dirty so the next checkpoint saves them clean."""
|
| 3378 |
try:
|
|
|
|
| 3737 |
con.execute(
|
| 3738 |
"INSERT OR REPLACE INTO _sync_state "
|
| 3739 |
"(entity, phase, last_id, cursor_wd, rows, updated_at) VALUES (?,?,?,?,?,?)",
|
| 3740 |
+
[str(key), str(phase), None, (str(detail) if str(key).startswith(("amzn_reconcile_", "amzn_limit_", "amzn_refused_", "amzn_pending_report_",
|
| 3741 |
+
"amzn_resync_"))
|
| 3742 |
else str(detail)[:300]) if detail else None, int(rows or 0),
|
| 3743 |
_iso(datetime.now(timezone.utc))])
|
| 3744 |
|
|
|
|
| 5345 |
except Exception as exc: # noqa: BLE001
|
| 5346 |
report["problems"].append(f"Amazon snapshot collapse failed ({type(exc).__name__}); the next "
|
| 5347 |
"resync of each snapshot table replaces its earlier copies.")
|
| 5348 |
+
# Backstop: a candidate reused across restarts never passed through the restore heal.
|
| 5349 |
try:
|
| 5350 |
changed_tables.update(_unquote_mirror_values(con, log))
|
| 5351 |
except Exception as exc: # noqa: BLE001
|
| 5352 |
report["problems"].append(f"Amazon quoted value heal failed ({type(exc).__name__}); the next "
|
| 5353 |
"resync of each quoted table writes clean values.")
|
| 5354 |
+
try:
|
| 5355 |
+
changed_tables.update(_collapse_identity(con, log))
|
| 5356 |
+
except Exception as exc: # noqa: BLE001
|
| 5357 |
+
report["problems"].append(f"Amazon identity collapse failed ({type(exc).__name__}); the next "
|
| 5358 |
+
"resync of each doubled table replaces its earlier document's rows.")
|
| 5359 |
|
| 5360 |
def write_window(table, cols, types, rows):
|
| 5361 |
if cx.REPORT_TYPES[table].get("weekly"):
|
|
|
|
| 5625 |
margin_windows = _month_windows(plan["start"], plan["end"])
|
| 5626 |
else:
|
| 5627 |
margin_windows = _windows(plan["start"], plan["end"], spec.get("max_window_days", MAX_WINDOW_DAYS))
|
| 5628 |
+
resync = None if first_load or spec.get("snapshot") else queued_resync(con, table)
|
| 5629 |
+
if resync is not None and read_watermark(con, table):
|
| 5630 |
+
# A queued window takes this tick; the watermark stays, so no new window is skipped.
|
| 5631 |
+
margin_windows = [resync]
|
| 5632 |
+
plan["end"] = read_watermark(con, table)
|
| 5633 |
+
plan["cause"] = f"resync of the stored window starting {_iso(resync[0])}"
|
| 5634 |
+
log(f"{table}: {plan['cause']}")
|
| 5635 |
+
elif table in GROUP_A_TABLES and not first_load and len(margin_windows) > 1:
|
| 5636 |
# One due window per table tick respects the vendor floor. The watermark
|
| 5637 |
# advances only to that window's end, so the following tick loses no history.
|
| 5638 |
remaining_windows = len(margin_windows) - 1
|
|
|
|
| 5655 |
{"state": "access", "operation": operation}, rt, report, log)
|
| 5656 |
count = _sync_margin_window(con, table, fields, w0, w1,
|
| 5657 |
max_wait=max_wait, log=log)
|
| 5658 |
+
if resync is not None and (w0, w1) == resync:
|
| 5659 |
+
resync_landed(con, table, resync)
|
| 5660 |
probe.finish(count)
|
| 5661 |
spend_rows(count)
|
| 5662 |
written += count
|
|
|
|
| 7084 |
whole["state"] == "refused" and whole["refused_tables"] == []
|
| 7085 |
and dump() == {"_sync_state": []}
|
| 7086 |
and not any(path.endswith(".parquet") for path in downloaded))
|
| 7087 |
+
|
| 7088 |
+
# S79-T19, S79-T23: a PUBLISHED snapshot and a hand-off heal the rows they serve.
|
| 7089 |
+
split_window = ("2026-03-20T18:01:25Z", "2026-04-19T18:01:25Z")
|
| 7090 |
+
|
| 7091 |
+
def seed_quoted():
|
| 7092 |
+
origin = "'GET_FIXTURE', 'Amazon Services API'"
|
| 7093 |
+
con.execute("CREATE TABLE amzn_pricing AS SELECT * FROM (VALUES "
|
| 7094 |
+
f"('P1', 'C1', 'S1', 5, 'doc-old', 1, {origin}), "
|
| 7095 |
+
f"('P1', 'C2', 'S2', 7, 'doc-old', 2, {origin}), "
|
| 7096 |
+
f"('P2', 'C1', 'S1', 9, 'doc-new', 1, {origin})) AS t(parent_asin, child_asin, sku, "
|
| 7097 |
+
"units, _report_document_id, _report_row_number, _report_type, _data_origin)")
|
| 7098 |
+
con.execute("CREATE TABLE amzn_inventory_ledger_detail AS SELECT * FROM (VALUES "
|
| 7099 |
+
"('2026-02-18T18:01:25Z', '2026-03-20T18:01:25Z', '\"SKU-1\"', '\"Plain\"', "
|
| 7100 |
+
"'\"Receipts\"', '\"12\"', 'doc-a', 1), "
|
| 7101 |
+
f"('{split_window[0]}', '{split_window[1]}', '\"SKU-2\"', '\"A vase', "
|
| 7102 |
+
"'with a tab\"', '\"Shipments\"', 'doc-b', 1)) AS t(_period_start, _period_end, msku, "
|
| 7103 |
+
"title, event_type, quantity, _report_document_id, _report_row_number)")
|
| 7104 |
+
|
| 7105 |
+
def served():
|
| 7106 |
+
return (con.execute("SELECT child_asin, parent_asin FROM amzn_pricing ORDER BY 1").fetchall(),
|
| 7107 |
+
con.execute("SELECT msku, quantity FROM amzn_inventory_ledger_detail "
|
| 7108 |
+
"ORDER BY _period_start").fetchall())
|
| 7109 |
+
|
| 7110 |
+
healed_rows = ([("C1", "P2"), ("C2", "P1")], [("SKU-1", "12"), ("SKU-2", "Shipments")])
|
| 7111 |
+
rt.blobs = {}
|
| 7112 |
+
wipe()
|
| 7113 |
+
restore()
|
| 7114 |
+
seed_quoted()
|
| 7115 |
+
record_watermark(con, "inventory_ledger_detail", datetime(2026, 10, 23, tzinfo=timezone.utc), 2)
|
| 7116 |
+
report = {"problems": []}
|
| 7117 |
+
_MIRROR_STORE.persist_mirror(rt.key, rt, con, {"amzn_pricing", "amzn_inventory_ledger_detail"},
|
| 7118 |
+
report=report, log=lambda *_a: None, _publication={
|
| 7119 |
+
"snapshot_id": uuid.uuid4().hex, "started_at": "2026-10-24T00:00:00Z",
|
| 7120 |
+
"state": "complete"})
|
| 7121 |
+
wipe()
|
| 7122 |
+
published = restore()
|
| 7123 |
+
ck("[restore] a published snapshot restore heals quoted and doubled rows, not marked dirty", lambda:
|
| 7124 |
+
not report["problems"] and published["state"] == "restored"
|
| 7125 |
+
and bool(published["manifest"].get("snapshot_id")) and not published["problems"]
|
| 7126 |
+
and not published["dirty"] and served() == healed_rows)
|
| 7127 |
+
ck("[restore] a split ledger row queues its own report window for download again", lambda:
|
| 7128 |
+
queued_resync(con, "inventory_ledger_detail") == (_utc(split_window[0]), _utc(split_window[1])))
|
| 7129 |
+
settled = dump()
|
| 7130 |
+
forget()
|
| 7131 |
+
again = restore()
|
| 7132 |
+
ck("[restore] the heal on a local restart of the same snapshot changes nothing", lambda:
|
| 7133 |
+
again["state"] == "local" and dump() == settled)
|
| 7134 |
+
resync_landed(con, "inventory_ledger_detail", queued_resync(con, "inventory_ledger_detail"))
|
| 7135 |
+
ck("[restore] a landed resync window leaves the queue", lambda:
|
| 7136 |
+
queued_resync(con, "inventory_ledger_detail") is None
|
| 7137 |
+
and _read_state(con, entity_key(RESYNC_PREFIX + "inventory_ledger_detail")) is None)
|
| 7138 |
+
|
| 7139 |
+
import tempfile as _tmp
|
| 7140 |
+
with _tmp.TemporaryDirectory(prefix="amazon-handoff-") as tmp:
|
| 7141 |
+
wipe()
|
| 7142 |
+
seed_quoted()
|
| 7143 |
+
paths = {}
|
| 7144 |
+
for name in ("amzn_pricing", "amzn_inventory_ledger_detail"):
|
| 7145 |
+
paths[name] = [Path(tmp) / (name + ".parquet")]
|
| 7146 |
+
con.execute(f"COPY {name} TO ? (FORMAT PARQUET)", [str(paths[name][0])])
|
| 7147 |
+
wipe()
|
| 7148 |
+
_mirrors.install_handoff("amazon", rt.key, rt, {"snapshot_id": uuid.uuid4().hex,
|
| 7149 |
+
"tables": {}}, [], paths)
|
| 7150 |
+
ck("[restore] a hand-off install heals the rows the web process serves", lambda:
|
| 7151 |
+
served() == healed_rows)
|
| 7152 |
+
|
| 7153 |
+
# A re-parented child replaces its own window's row; another window keeps its own.
|
| 7154 |
+
con.execute("DROP TABLE amzn_pricing")
|
| 7155 |
+
|
| 7156 |
+
def land_pricing(doc, parent, start):
|
| 7157 |
+
w0 = datetime(2026, 10, start, tzinfo=timezone.utc)
|
| 7158 |
+
rows = [_origin({"parent_asin": parent, "child_asin": "C1", "sku": "S1", "units": 3},
|
| 7159 |
+
doc, 1, "GET_SALES_AND_TRAFFIC_REPORT")]
|
| 7160 |
+
cols, rows = stamp_week(["parent_asin", "child_asin", "sku", "units", *_ORIGIN_TYPES],
|
| 7161 |
+
rows, w0, w0 + timedelta(days=7))
|
| 7162 |
+
types = _ensure(con, "amzn_pricing", cols, rows)
|
| 7163 |
+
return replace_window(con, "amzn_pricing", cols, types, rows, replace_keys_for("pricing"))
|
| 7164 |
+
|
| 7165 |
+
landed = [land_pricing("doc-1", "P1", 1), land_pricing("doc-2", "P2", 1)]
|
| 7166 |
+
ck("[restore] a re-parented pricing child replaces its older document's row", lambda:
|
| 7167 |
+
landed == [(1, None), (1, None)] and con.execute(
|
| 7168 |
+
"SELECT parent_asin, _report_document_id FROM amzn_pricing").fetchall() == [("P2", "doc-2")])
|
| 7169 |
+
land_pricing("doc-3", "P2", 8)
|
| 7170 |
+
ck("[restore] a later pricing window keeps its own row beside the earlier window", lambda:
|
| 7171 |
+
con.execute("SELECT count(*), count(DISTINCT _period_start) FROM amzn_pricing").fetchone() == (2, 2))
|
| 7172 |
finally:
|
| 7173 |
forget()
|
| 7174 |
con.close()
|
|
|
|
| 7967 |
con.execute("DELETE FROM _sync_state WHERE entity=?", [entity_key(refused_table)])
|
| 7968 |
ck("NC: the same table never refused (no watermark) still owes a first load",
|
| 7969 |
needs_first_load(rt.key, rt))
|
| 7970 |
+
# S79-T23: a queued resync window downloads again alone; the watermark stays.
|
| 7971 |
+
fees = "amzn_long_term_storage_fees"
|
| 7972 |
+
stored = con.execute(f"SELECT _period_start, _period_end, count(*) FROM {fees} "
|
| 7973 |
+
"GROUP BY 1, 2 ORDER BY 1").fetchall()
|
| 7974 |
+
mark = read_watermark(con, "long_term_storage_fees")
|
| 7975 |
+
_record_state(con, entity_key(RESYNC_PREFIX + "long_term_storage_fees"), "pending", 1,
|
| 7976 |
+
json.dumps({"windows": [list(stored[0][:2])]}))
|
| 7977 |
+
clock[0] += timedelta(days=2)
|
| 7978 |
+
before = len(calls)
|
| 7979 |
+
_sync_candidate(rt.key, tables=("long_term_storage_fees",), rt=rt, log=lambda *_: None)
|
| 7980 |
+
asked = [kwargs for _report, kwargs in calls[before:]]
|
| 7981 |
+
ck("a queued resync window downloads again alone and keeps the watermark",
|
| 7982 |
+
len(asked) == 1 and str(asked[0].get("start", ""))[:10] == stored[0][0][:10]
|
| 7983 |
+
and read_watermark(con, "long_term_storage_fees") == mark
|
| 7984 |
+
and queued_resync(con, "long_term_storage_fees") is None
|
| 7985 |
+
and con.execute(f"SELECT _period_start, _period_end, count(*) FROM {fees} "
|
| 7986 |
+
"GROUP BY 1, 2 ORDER BY 1").fetchall() == stored)
|
| 7987 |
finally:
|
| 7988 |
con.close()
|
| 7989 |
|
platform/harness/mirror_store.py
CHANGED
|
@@ -93,6 +93,9 @@ def install_handoff(connector, tenant_key, rt, manifest, rows, paths):
|
|
| 93 |
return False
|
| 94 |
only = changed_tables(previous, manifest, store._tables(con), paths)
|
| 95 |
store._install_snapshot(con, manifest, [tuple(row) for row in rows], paths, only=only)
|
|
|
|
|
|
|
|
|
|
| 96 |
newest = _record_newest(con, store.connector)
|
| 97 |
finally:
|
| 98 |
con.close()
|
|
@@ -453,6 +456,10 @@ class MirrorStore:
|
|
| 453 |
def _after_restore(self, con, result, log):
|
| 454 |
pass
|
| 455 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 456 |
def _restored_state_rows(self, rows, tables, refused):
|
| 457 |
# Without a connector-specific marker policy, invalidate all its sync
|
| 458 |
# decisions after any table refusal. Good tables remain available.
|
|
@@ -728,6 +735,8 @@ class MirrorStore:
|
|
| 728 |
result["manifest"] = saved
|
| 729 |
if not result["manifest"].get("snapshot_id"):
|
| 730 |
self._after_restore(con, result, log)
|
|
|
|
|
|
|
| 731 |
with _CATCH_UP_LOCK:
|
| 732 |
_catch_up_source(rt, self.connector)["pass_pending"] = True
|
| 733 |
self.cache[cache_key] = result
|
|
@@ -759,6 +768,10 @@ class MirrorStore:
|
|
| 759 |
self._log_restored(paths, total, rows, log)
|
| 760 |
if not manifest.get("snapshot_id"):
|
| 761 |
self._after_restore(con, result, log)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 762 |
table_states = {row[0]: row[1] for row in rows if row[0] in paths}
|
| 763 |
current = (not refused and bool(paths) and set(paths) <= set(table_states)
|
| 764 |
and all(row[1] in {"live", "done"} for row in rows))
|
|
|
|
| 93 |
return False
|
| 94 |
only = changed_tables(previous, manifest, store._tables(con), paths)
|
| 95 |
store._install_snapshot(con, manifest, [tuple(row) for row in rows], paths, only=only)
|
| 96 |
+
# The web process serves the hand-off as installed, so it heals its own copy too.
|
| 97 |
+
healed = {"dirty": set(), "problems": []}
|
| 98 |
+
store._heal_restored(con, healed, print)
|
| 99 |
newest = _record_newest(con, store.connector)
|
| 100 |
finally:
|
| 101 |
con.close()
|
|
|
|
| 456 |
def _after_restore(self, con, result, log):
|
| 457 |
pass
|
| 458 |
|
| 459 |
+
def _heal_restored(self, con, result, log):
|
| 460 |
+
"""Row heals for a published snapshot or hand-off: idempotent, one probe per table when clean."""
|
| 461 |
+
pass
|
| 462 |
+
|
| 463 |
def _restored_state_rows(self, rows, tables, refused):
|
| 464 |
# Without a connector-specific marker policy, invalidate all its sync
|
| 465 |
# decisions after any table refusal. Good tables remain available.
|
|
|
|
| 735 |
result["manifest"] = saved
|
| 736 |
if not result["manifest"].get("snapshot_id"):
|
| 737 |
self._after_restore(con, result, log)
|
| 738 |
+
else:
|
| 739 |
+
self._heal_restored(con, result, log)
|
| 740 |
with _CATCH_UP_LOCK:
|
| 741 |
_catch_up_source(rt, self.connector)["pass_pending"] = True
|
| 742 |
self.cache[cache_key] = result
|
|
|
|
| 768 |
self._log_restored(paths, total, rows, log)
|
| 769 |
if not manifest.get("snapshot_id"):
|
| 770 |
self._after_restore(con, result, log)
|
| 771 |
+
else:
|
| 772 |
+
# Trap: the healed local rows differ from the published snapshot until the
|
| 773 |
+
# next publish; each restore heals again and a second heal changes nothing.
|
| 774 |
+
self._heal_restored(con, result, log)
|
| 775 |
table_states = {row[0]: row[1] for row in rows if row[0] in paths}
|
| 776 |
current = (not refused and bool(paths) and set(paths) <= set(table_states)
|
| 777 |
and all(row[1] in {"live", "done"} for row in rows))
|
platform/modules/opportunity_finder.py
CHANGED
|
@@ -19,8 +19,11 @@ A rank is a position: a lower number is searched more, so a term rises when its
|
|
| 19 |
The department filter and the rank comparison run in DuckDB over every row, with no cap (rule 1).
|
| 20 |
"""
|
| 21 |
import contextlib
|
|
|
|
| 22 |
import datetime as dt
|
| 23 |
import hashlib
|
|
|
|
|
|
|
| 24 |
from pathlib import Path
|
| 25 |
|
| 26 |
APP_LABEL = "Opportunity finder"
|
|
@@ -45,6 +48,11 @@ TRIGGER = "brand_analytics"
|
|
| 45 |
|
| 46 |
NO_MIRROR = "The Amazon mirror is not on this server yet. It appears after the first sync."
|
| 47 |
NOT_SYNCED = "ut_amzn_brand_analytics has not synced yet, so no search term can be compared."
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 48 |
NO_DEPARTMENTS = ("No brand analytics row shows one of your ASINs yet, so the finder has no department to "
|
| 49 |
"search. It reads departments from ut_amzn_brand_analytics and your ASINs from "
|
| 50 |
"ut_amzn_search_query_performance and ut_amzn_listings.")
|
|
@@ -58,6 +66,11 @@ OLDER_WEEK = ("Only the newest brand analytics week can be tied. The mirror keep
|
|
| 58 |
LIMIT = ("Each week of history is one store document, limited to 128 MB. About 3,000,000 search terms "
|
| 59 |
"fit in one week. Split a week by department when a marketplace nears that size.")
|
| 60 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 61 |
#: Accounts that sign in for a process, never a person who can own an item.
|
| 62 |
SERVICE_ACCOUNTS = ("admin", "automation", "scheduler", "qa-runner", "leadership")
|
| 63 |
|
|
@@ -115,36 +128,126 @@ def _mirror_path(tenant_key, st):
|
|
| 115 |
return amazon_store._tenant_mirror_path(tenant_key, rt)
|
| 116 |
|
| 117 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 118 |
@contextlib.contextmanager
|
| 119 |
def _reader(tenant_key, st):
|
| 120 |
-
"""A read
|
| 121 |
-
|
| 122 |
-
|
| 123 |
-
|
| 124 |
-
return
|
| 125 |
from harness import datastore
|
|
|
|
| 126 |
try:
|
| 127 |
shared = Path(datastore.DB_PATH).resolve() == path.resolve()
|
| 128 |
except Exception: # noqa: BLE001
|
| 129 |
shared = False
|
| 130 |
-
|
| 131 |
-
|
| 132 |
-
|
| 133 |
-
|
| 134 |
-
|
| 135 |
-
|
| 136 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 137 |
return
|
| 138 |
-
|
| 139 |
-
con = duckdb.connect(str(path), read_only=True)
|
| 140 |
try:
|
| 141 |
-
yield
|
| 142 |
finally:
|
| 143 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 144 |
|
| 145 |
|
| 146 |
def _columns(cur, table):
|
| 147 |
-
if cur is None:
|
| 148 |
return set()
|
| 149 |
try:
|
| 150 |
return {r[0] for r in cur.execute(
|
|
@@ -153,6 +256,15 @@ def _columns(cur, table):
|
|
| 153 |
return set()
|
| 154 |
|
| 155 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 156 |
def _our_asins(cur):
|
| 157 |
"""SQP ASINs plus listing `asin1`: the ASINs the seller owns."""
|
| 158 |
ours = set()
|
|
@@ -165,10 +277,14 @@ def _our_asins(cur):
|
|
| 165 |
return {a for a in ours if a}
|
| 166 |
|
| 167 |
|
| 168 |
-
def _period(cur):
|
| 169 |
-
"""The newest brand analytics `_period_start` as text, or None.
|
| 170 |
-
|
| 171 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 172 |
|
| 173 |
|
| 174 |
_TERM = 'lower(trim(CAST("search_term" AS VARCHAR)))'
|
|
@@ -188,21 +304,46 @@ def _derived_departments(cur, ours):
|
|
| 188 |
return sorted(_text(r[0]) for r in rows if _text(r[0]))
|
| 189 |
|
| 190 |
|
| 191 |
-
def
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 192 |
"""What the mirror holds now: {period, departments, ours, problems}. Departments follow the setting."""
|
| 193 |
out = {"period": None, "departments": [], "ours": set(), "problems": [], "derived": []}
|
| 194 |
if cur is None:
|
| 195 |
out["problems"].append(NO_MIRROR)
|
| 196 |
return out
|
| 197 |
-
|
| 198 |
-
|
|
|
|
|
|
|
| 199 |
return out
|
| 200 |
-
|
| 201 |
-
if out["period"] is None:
|
| 202 |
out["problems"].append(NOT_SYNCED)
|
| 203 |
return out
|
| 204 |
-
|
| 205 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 206 |
chosen = _doc(st, KEY_SETTINGS).get("departments")
|
| 207 |
out["departments"] = sorted({_text(d) for d in chosen if _text(d)}) if chosen else out["derived"]
|
| 208 |
if not out["departments"]:
|
|
@@ -247,7 +388,10 @@ def _rising_rows(cur, period, departments, prior, min_pct):
|
|
| 247 |
f'AND (prior.rank - cur.rank) * 100.0 / prior.rank >= ? '
|
| 248 |
f'ORDER BY pct DESC, cur.term')
|
| 249 |
params = [period, list(departments), [t for t, _r in pairs], [r for _t, r in pairs], *params_top, float(min_pct)]
|
| 250 |
-
|
|
|
|
|
|
|
|
|
|
| 251 |
|
| 252 |
|
| 253 |
def _top3_list(raw, ours):
|
|
@@ -371,7 +515,8 @@ def record_week(tenant_key, *, st=None, now=None):
|
|
| 371 |
st = _rt(tenant_key, st)
|
| 372 |
now = now or _now()
|
| 373 |
with _reader(tenant_key, st) as cur:
|
| 374 |
-
|
|
|
|
| 375 |
if src["period"] is None or not src["departments"]:
|
| 376 |
return {"state": "nothing", "problems": src["problems"]}
|
| 377 |
terms = week_ranks(cur, src["period"], src["departments"])
|
|
@@ -434,6 +579,7 @@ def find(tenant_key, *, st=None):
|
|
| 434 |
error = _doc(st, KEY_INDEX).get("last_error")
|
| 435 |
if error:
|
| 436 |
problems.append(error)
|
|
|
|
| 437 |
found = []
|
| 438 |
for term, before, now_rank, rise, pct, top in rows:
|
| 439 |
top3 = _top3_list(top, src["ours"])
|
|
@@ -441,10 +587,23 @@ def find(tenant_key, *, st=None):
|
|
| 441 |
found.append({"id": item_id(term), "term": term, "title": term, "priorRank": int(before),
|
| 442 |
"rank": int(now_rank), "rise": int(rise), "risePct": round(float(pct), 2),
|
| 443 |
"ours": bool(mine), "asin": mine[0] if mine else None, "top3": top3, "rising": True,
|
|
|
|
|
|
|
| 444 |
"reason": reason_for(int(before), int(now_rank), float(pct), bool(mine))})
|
| 445 |
return {"week": src["period"][:10] if src["period"] else None, "period": src["period"],
|
| 446 |
"priorWeek": prior_week, "departments": src["departments"], "derivedDepartments": src["derived"],
|
| 447 |
-
"rising": found, "problems": problems}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 448 |
|
| 449 |
|
| 450 |
def _stored(st):
|
|
@@ -499,7 +658,7 @@ def items(tenant_key, user=None, *, status=None, cursor=None, limit=DEFAULT_LIMI
|
|
| 499 |
page, nxt = _page(wanted, cursor, limit)
|
| 500 |
return {"items": page, "cursor": nxt, "counts": counts, "total": len(wanted), "week": found["week"],
|
| 501 |
"priorWeek": found["priorWeek"], "departments": found["departments"],
|
| 502 |
-
"problems": found["problems"], "weeks": _index(st)}
|
| 503 |
|
| 504 |
|
| 505 |
def _lookup(st, iid, found=None):
|
|
@@ -643,7 +802,7 @@ def validate(tenant_key, *, week=None, start=None, end=None, st=None):
|
|
| 643 |
"""Rule 8: the stored week's term count against a SQL distinct count of the mirror week, our departments."""
|
| 644 |
st = _rt(tenant_key, st)
|
| 645 |
with _reader(tenant_key, st) as cur:
|
| 646 |
-
src = _source(cur, st)
|
| 647 |
period = src["period"]
|
| 648 |
if period is None:
|
| 649 |
return {"ok": False, "app_total": None, "independent_total": None, "week": None,
|
|
|
|
| 19 |
The department filter and the rank comparison run in DuckDB over every row, with no cap (rule 1).
|
| 20 |
"""
|
| 21 |
import contextlib
|
| 22 |
+
import contextvars
|
| 23 |
import datetime as dt
|
| 24 |
import hashlib
|
| 25 |
+
import os
|
| 26 |
+
import threading
|
| 27 |
from pathlib import Path
|
| 28 |
|
| 29 |
APP_LABEL = "Opportunity finder"
|
|
|
|
| 48 |
|
| 49 |
NO_MIRROR = "The Amazon mirror is not on this server yet. It appears after the first sync."
|
| 50 |
NOT_SYNCED = "ut_amzn_brand_analytics has not synced yet, so no search term can be compared."
|
| 51 |
+
#: The mirror holds the table but this read failed or was busy. `{cause}` is the error class.
|
| 52 |
+
READ_FAILED = ("The Amazon mirror could not be read just now ({cause}), so no search term can be compared. "
|
| 53 |
+
"Try again in a minute.")
|
| 54 |
+
BAD_PERIOD = ("The newest brand analytics week start reads {value}, which is not an ISO date, so no week "
|
| 55 |
+
"can be compared.")
|
| 56 |
NO_DEPARTMENTS = ("No brand analytics row shows one of your ASINs yet, so the finder has no department to "
|
| 57 |
"search. It reads departments from ut_amzn_brand_analytics and your ASINs from "
|
| 58 |
"ut_amzn_search_query_performance and ut_amzn_listings.")
|
|
|
|
| 66 |
LIMIT = ("Each week of history is one store document, limited to 128 MB. About 3,000,000 search terms "
|
| 67 |
"fit in one week. Split a week by department when a marketplace nears that size.")
|
| 68 |
|
| 69 |
+
#: A mirror probe longer than this is interrupted and reads as busy, under the 45 s request deadline.
|
| 70 |
+
PROBE_SECONDS = 20.0
|
| 71 |
+
#: Mirror facts and rising lists kept per process, keyed on the mirror mark. Oldest goes first.
|
| 72 |
+
MEMO_KEEP = 16
|
| 73 |
+
|
| 74 |
#: Accounts that sign in for a process, never a person who can own an item.
|
| 75 |
SERVICE_ACCOUNTS = ("admin", "automation", "scheduler", "qa-runner", "leadership")
|
| 76 |
|
|
|
|
| 128 |
return amazon_store._tenant_mirror_path(tenant_key, rt)
|
| 129 |
|
| 130 |
|
| 131 |
+
class _Unreadable:
|
| 132 |
+
"""The mirror exists but no cursor could be opened on it. `cause` names the error class."""
|
| 133 |
+
|
| 134 |
+
def __init__(self, cause):
|
| 135 |
+
self.cause = cause
|
| 136 |
+
|
| 137 |
+
|
| 138 |
+
class _ReadFailed(Exception):
|
| 139 |
+
"""A mirror probe failed or was interrupted. Never read as an absent table."""
|
| 140 |
+
|
| 141 |
+
|
| 142 |
+
#: The mirror mark of the cursor `_reader` yielded: (path, install generation, file stat).
|
| 143 |
+
_MARK = contextvars.ContextVar("opportunity_finder_mark", default=None)
|
| 144 |
+
_MEMO = {}
|
| 145 |
+
_MEMO_LOCK = threading.Lock()
|
| 146 |
+
|
| 147 |
+
|
| 148 |
+
def _remember(key, build):
|
| 149 |
+
"""`build()` once per key. A failed build is not kept, so the next request reads again."""
|
| 150 |
+
if key is None:
|
| 151 |
+
return build()
|
| 152 |
+
with _MEMO_LOCK:
|
| 153 |
+
if key in _MEMO:
|
| 154 |
+
return _MEMO[key]
|
| 155 |
+
value = build()
|
| 156 |
+
with _MEMO_LOCK:
|
| 157 |
+
_MEMO[key] = value
|
| 158 |
+
while len(_MEMO) > MEMO_KEEP:
|
| 159 |
+
_MEMO.pop(next(iter(_MEMO)))
|
| 160 |
+
return value
|
| 161 |
+
|
| 162 |
+
|
| 163 |
+
def _mark(path, shared):
|
| 164 |
+
"""Moves whenever the mirror rows can have changed: an install, a file swap or a rewrite."""
|
| 165 |
+
from harness import datastore
|
| 166 |
+
generation = None
|
| 167 |
+
if shared:
|
| 168 |
+
with contextlib.suppress(Exception):
|
| 169 |
+
generation = datastore.mirror_generation()
|
| 170 |
+
stamps = []
|
| 171 |
+
for part in (Path(path), Path(str(path) + ".wal")):
|
| 172 |
+
try:
|
| 173 |
+
stat = os.stat(part)
|
| 174 |
+
stamps.append((stat.st_ino, stat.st_mtime_ns, stat.st_size))
|
| 175 |
+
except OSError:
|
| 176 |
+
stamps.append(None)
|
| 177 |
+
return (str(path), generation, tuple(stamps))
|
| 178 |
+
|
| 179 |
+
|
| 180 |
@contextlib.contextmanager
|
| 181 |
def _reader(tenant_key, st):
|
| 182 |
+
"""A read cursor on the tenant mirror, None when there is no mirror, `_Unreadable` when it cannot open.
|
| 183 |
+
|
| 184 |
+
The web process reads through the same door as the typed `ut_amzn_*` reads: assert the tenant, then
|
| 185 |
+
`datastore.ro_cursor()` on the serving instance, which follows the installed mirror version."""
|
|
|
|
| 186 |
from harness import datastore
|
| 187 |
+
path = Path(_mirror_path(tenant_key, st))
|
| 188 |
try:
|
| 189 |
shared = Path(datastore.DB_PATH).resolve() == path.resolve()
|
| 190 |
except Exception: # noqa: BLE001
|
| 191 |
shared = False
|
| 192 |
+
if not path.exists():
|
| 193 |
+
yield None
|
| 194 |
+
return
|
| 195 |
+
cur = None
|
| 196 |
+
try:
|
| 197 |
+
if shared:
|
| 198 |
+
if hasattr(st, "assert_datastore_matches"):
|
| 199 |
+
st.assert_datastore_matches()
|
| 200 |
+
cur = datastore.ro_cursor()
|
| 201 |
+
else:
|
| 202 |
+
import duckdb
|
| 203 |
+
cur = duckdb.connect(str(path), read_only=True)
|
| 204 |
+
except Exception as exc: # noqa: BLE001
|
| 205 |
+
if cur is not None:
|
| 206 |
+
with contextlib.suppress(Exception):
|
| 207 |
+
cur.close()
|
| 208 |
+
yield _Unreadable(type(exc).__name__)
|
| 209 |
return
|
| 210 |
+
token = _MARK.set(_mark(path, shared))
|
|
|
|
| 211 |
try:
|
| 212 |
+
yield cur
|
| 213 |
finally:
|
| 214 |
+
_MARK.reset(token)
|
| 215 |
+
cur.close()
|
| 216 |
+
|
| 217 |
+
|
| 218 |
+
def _interrupt(cur):
|
| 219 |
+
with contextlib.suppress(Exception):
|
| 220 |
+
cur.interrupt()
|
| 221 |
+
|
| 222 |
+
|
| 223 |
+
def _bounded(cur, sql, params=()):
|
| 224 |
+
"""Every row of one probe query, interrupted past PROBE_SECONDS. Any failure raises `_ReadFailed`.
|
| 225 |
+
|
| 226 |
+
Pass a query without parameters: DuckDB ignores interrupt() on a parameterized execute."""
|
| 227 |
+
if cur is None or isinstance(cur, _Unreadable):
|
| 228 |
+
raise _ReadFailed(getattr(cur, "cause", "NoMirror"))
|
| 229 |
+
timer = threading.Timer(PROBE_SECONDS, _interrupt, [cur])
|
| 230 |
+
timer.daemon = True
|
| 231 |
+
timer.start()
|
| 232 |
+
try:
|
| 233 |
+
return cur.execute(sql, list(params)).fetchall()
|
| 234 |
+
except Exception as exc: # noqa: BLE001
|
| 235 |
+
raise _ReadFailed(type(exc).__name__) from exc
|
| 236 |
+
finally:
|
| 237 |
+
timer.cancel()
|
| 238 |
+
|
| 239 |
+
|
| 240 |
+
def _types(cur, table):
|
| 241 |
+
"""{column: data type} of a mirror table, {} when the table is absent. A failed read raises."""
|
| 242 |
+
if not table.replace("_", "").isalnum():
|
| 243 |
+
raise ValueError(f"{table!r} is not a table name.")
|
| 244 |
+
# A literal, not a parameter: DuckDB ignores interrupt() on a parameterized execute.
|
| 245 |
+
return {str(n): str(t) for n, t in _bounded(
|
| 246 |
+
cur, f"SELECT column_name, data_type FROM information_schema.columns WHERE table_name = '{table}'")}
|
| 247 |
|
| 248 |
|
| 249 |
def _columns(cur, table):
|
| 250 |
+
if cur is None or isinstance(cur, _Unreadable):
|
| 251 |
return set()
|
| 252 |
try:
|
| 253 |
return {r[0] for r in cur.execute(
|
|
|
|
| 256 |
return set()
|
| 257 |
|
| 258 |
|
| 259 |
+
def iso_period(value):
|
| 260 |
+
"""True when a `_period_start` text starts with an ISO date (YYYY-MM-DD), as `stamp_week` writes it."""
|
| 261 |
+
try:
|
| 262 |
+
dt.date.fromisoformat(_text(value)[:10])
|
| 263 |
+
except ValueError:
|
| 264 |
+
return False
|
| 265 |
+
return len(_text(value)) >= 10
|
| 266 |
+
|
| 267 |
+
|
| 268 |
def _our_asins(cur):
|
| 269 |
"""SQP ASINs plus listing `asin1`: the ASINs the seller owns."""
|
| 270 |
ours = set()
|
|
|
|
| 277 |
return {a for a in ours if a}
|
| 278 |
|
| 279 |
|
| 280 |
+
def _period(cur, kind="VARCHAR"):
|
| 281 |
+
"""The newest brand analytics `_period_start` as text, or None. One column scan, bounded.
|
| 282 |
+
|
| 283 |
+
A text column takes MAX as stored. A DATE or TIMESTAMP column casts only the one MAX value."""
|
| 284 |
+
newest = ('MAX("_period_start")' if str(kind).upper().startswith("VARCHAR")
|
| 285 |
+
else 'CAST(MAX("_period_start") AS VARCHAR)')
|
| 286 |
+
rows = _bounded(cur, f'SELECT {newest} FROM {BA_TABLE}')
|
| 287 |
+
return str(rows[0][0]) if rows and rows[0][0] is not None else None
|
| 288 |
|
| 289 |
|
| 290 |
_TERM = 'lower(trim(CAST("search_term" AS VARCHAR)))'
|
|
|
|
| 304 |
return sorted(_text(r[0]) for r in rows if _text(r[0]))
|
| 305 |
|
| 306 |
|
| 307 |
+
def _facts(cur, fresh=False):
|
| 308 |
+
"""The mirror facts every read starts from, once per mirror mark: columns, newest period, ours, derived."""
|
| 309 |
+
def build():
|
| 310 |
+
types = _types(cur, BA_TABLE)
|
| 311 |
+
facts = {"columns": set(types), "period": None, "ours": set(), "derived": []}
|
| 312 |
+
if not set(BA_NEED) <= facts["columns"]:
|
| 313 |
+
return facts
|
| 314 |
+
facts["period"] = _period(cur, types.get("_period_start", "VARCHAR"))
|
| 315 |
+
if facts["period"] is None or not iso_period(facts["period"]):
|
| 316 |
+
return facts
|
| 317 |
+
try:
|
| 318 |
+
facts["ours"] = _our_asins(cur)
|
| 319 |
+
facts["derived"] = _derived_departments(cur, facts["ours"])
|
| 320 |
+
except Exception as exc: # noqa: BLE001
|
| 321 |
+
raise _ReadFailed(type(exc).__name__) from exc
|
| 322 |
+
return facts
|
| 323 |
+
mark = _MARK.get()
|
| 324 |
+
return _remember(None if mark is None or fresh else ("facts", mark), build)
|
| 325 |
+
|
| 326 |
+
|
| 327 |
+
def _source(cur, st, fresh=False):
|
| 328 |
"""What the mirror holds now: {period, departments, ours, problems}. Departments follow the setting."""
|
| 329 |
out = {"period": None, "departments": [], "ours": set(), "problems": [], "derived": []}
|
| 330 |
if cur is None:
|
| 331 |
out["problems"].append(NO_MIRROR)
|
| 332 |
return out
|
| 333 |
+
try:
|
| 334 |
+
facts = _facts(cur, fresh)
|
| 335 |
+
except _ReadFailed as exc:
|
| 336 |
+
out["problems"].append(READ_FAILED.format(cause=exc))
|
| 337 |
return out
|
| 338 |
+
if not set(BA_NEED) <= facts["columns"] or facts["period"] is None:
|
|
|
|
| 339 |
out["problems"].append(NOT_SYNCED)
|
| 340 |
return out
|
| 341 |
+
if not iso_period(facts["period"]):
|
| 342 |
+
out["problems"].append(BAD_PERIOD.format(value=facts["period"]))
|
| 343 |
+
return out
|
| 344 |
+
out["period"] = facts["period"]
|
| 345 |
+
out["ours"] = set(facts["ours"])
|
| 346 |
+
out["derived"] = list(facts["derived"])
|
| 347 |
chosen = _doc(st, KEY_SETTINGS).get("departments")
|
| 348 |
out["departments"] = sorted({_text(d) for d in chosen if _text(d)}) if chosen else out["derived"]
|
| 349 |
if not out["departments"]:
|
|
|
|
| 388 |
f'AND (prior.rank - cur.rank) * 100.0 / prior.rank >= ? '
|
| 389 |
f'ORDER BY pct DESC, cur.term')
|
| 390 |
params = [period, list(departments), [t for t, _r in pairs], [r for _t, r in pairs], *params_top, float(min_pct)]
|
| 391 |
+
mark = _MARK.get()
|
| 392 |
+
key = (None if mark is None else
|
| 393 |
+
("rising", mark, period, tuple(departments), float(min_pct), hash(tuple(pairs))))
|
| 394 |
+
return _remember(key, lambda: cur.execute(sql, params).fetchall())
|
| 395 |
|
| 396 |
|
| 397 |
def _top3_list(raw, ours):
|
|
|
|
| 515 |
st = _rt(tenant_key, st)
|
| 516 |
now = now or _now()
|
| 517 |
with _reader(tenant_key, st) as cur:
|
| 518 |
+
# A sync just wrote the mirror, so the period is read again, never from the memo.
|
| 519 |
+
src = _source(cur, st, fresh=True)
|
| 520 |
if src["period"] is None or not src["departments"]:
|
| 521 |
return {"state": "nothing", "problems": src["problems"]}
|
| 522 |
terms = week_ranks(cur, src["period"], src["departments"])
|
|
|
|
| 579 |
error = _doc(st, KEY_INDEX).get("last_error")
|
| 580 |
if error:
|
| 581 |
problems.append(error)
|
| 582 |
+
windows = _windows(st, src["period"], prior_week)
|
| 583 |
found = []
|
| 584 |
for term, before, now_rank, rise, pct, top in rows:
|
| 585 |
top3 = _top3_list(top, src["ours"])
|
|
|
|
| 587 |
found.append({"id": item_id(term), "term": term, "title": term, "priorRank": int(before),
|
| 588 |
"rank": int(now_rank), "rise": int(rise), "risePct": round(float(pct), 2),
|
| 589 |
"ours": bool(mine), "asin": mine[0] if mine else None, "top3": top3, "rising": True,
|
| 590 |
+
"periodStart": src["period"], "priorPeriodStart": windows[1]["periodStart"] if len(windows) > 1
|
| 591 |
+
else None,
|
| 592 |
"reason": reason_for(int(before), int(now_rank), float(pct), bool(mine))})
|
| 593 |
return {"week": src["period"][:10] if src["period"] else None, "period": src["period"],
|
| 594 |
"priorWeek": prior_week, "departments": src["departments"], "derivedDepartments": src["derived"],
|
| 595 |
+
"windows": windows, "rising": found, "problems": problems}
|
| 596 |
+
|
| 597 |
+
|
| 598 |
+
def _windows(st, period, prior_week):
|
| 599 |
+
"""The compared weeks, newest first: each window's week key and its `_period_start` text."""
|
| 600 |
+
out = []
|
| 601 |
+
if period:
|
| 602 |
+
out.append({"window": "current", "week": period[:10], "periodStart": period, "iso": iso_period(period)})
|
| 603 |
+
if prior_week:
|
| 604 |
+
text = _text(_doc(st, WEEK_PREFIX + prior_week).get("period")) or prior_week
|
| 605 |
+
out.append({"window": "prior", "week": prior_week, "periodStart": text, "iso": iso_period(text)})
|
| 606 |
+
return out
|
| 607 |
|
| 608 |
|
| 609 |
def _stored(st):
|
|
|
|
| 658 |
page, nxt = _page(wanted, cursor, limit)
|
| 659 |
return {"items": page, "cursor": nxt, "counts": counts, "total": len(wanted), "week": found["week"],
|
| 660 |
"priorWeek": found["priorWeek"], "departments": found["departments"],
|
| 661 |
+
"problems": found["problems"], "weeks": _index(st), "windows": found["windows"]}
|
| 662 |
|
| 663 |
|
| 664 |
def _lookup(st, iid, found=None):
|
|
|
|
| 802 |
"""Rule 8: the stored week's term count against a SQL distinct count of the mirror week, our departments."""
|
| 803 |
st = _rt(tenant_key, st)
|
| 804 |
with _reader(tenant_key, st) as cur:
|
| 805 |
+
src = _source(cur, st, fresh=True)
|
| 806 |
period = src["period"]
|
| 807 |
if period is None:
|
| 808 |
return {"ok": False, "app_total": None, "independent_total": None, "week": None,
|
web/public/AIOS_DEPLOY_RECEIPT
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
|
|
|
|
| 1 |
+
7d1ba4682927d550a33ca880be112e51699b4a79459a302274e1e09cd13ef434
|
web/src/apps/catalog/CatalogApp.tsx
CHANGED
|
@@ -24,11 +24,12 @@ export function CatalogApp({ definition, database, scope, onOpen, onData, editor
|
|
| 24 |
useEffect(() => {
|
| 25 |
if (!data.loading && !data.error) onData?.(data.fields, data.allRows);
|
| 26 |
}, [data.loading, data.error, data.fields, data.allRows, onData]);
|
|
|
|
| 27 |
if (data.loading) {
|
| 28 |
-
return <
|
| 29 |
-
<
|
| 30 |
-
|
| 31 |
-
</
|
| 32 |
}
|
| 33 |
if (data.error) {
|
| 34 |
return <div className="cg-app-launch-state is-error" role="alert">
|
|
|
|
| 24 |
useEffect(() => {
|
| 25 |
if (!data.loading && !data.error) onData?.(data.fields, data.allRows);
|
| 26 |
}, [data.loading, data.error, data.fields, data.allRows, onData]);
|
| 27 |
+
// The Catalogs list reads its own records, so it paints while rows load. Only the editor waits.
|
| 28 |
if (data.loading) {
|
| 29 |
+
return <section className="cg-app-catalog is-print-editor" aria-label={`${definition.name} catalog`}>
|
| 30 |
+
<CatalogApplicationView definition={definition} rows={data.rows} onOpen={onOpen}
|
| 31 |
+
editor={{ canEdit: editor.canEdit }} rowsPending={{ read: data.progress }} />
|
| 32 |
+
</section>;
|
| 33 |
}
|
| 34 |
if (data.error) {
|
| 35 |
return <div className="cg-app-launch-state is-error" role="alert">
|
web/src/customer-grid/CatalogView.tsx
CHANGED
|
@@ -221,17 +221,29 @@ export interface CatalogApplicationEditor {
|
|
| 221 |
onSave: (catalogs: CatalogSpec[], catalogSpec: CatalogViewSpec) => Promise<void>;
|
| 222 |
}
|
| 223 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 224 |
/** Apps read C1 records. Sheets retain their read-only row projection. */
|
| 225 |
export const CatalogApplicationView = memo(function CatalogApplicationView({
|
| 226 |
definition,
|
| 227 |
rows,
|
| 228 |
onOpen,
|
| 229 |
editor,
|
|
|
|
| 230 |
}: {
|
| 231 |
definition: CatalogApplicationDefinition;
|
| 232 |
rows: Row[];
|
| 233 |
onOpen: (pid: number) => void;
|
| 234 |
-
editor?: CatalogApplicationEditor;
|
|
|
|
| 235 |
}) {
|
| 236 |
const items = useMemo(
|
| 237 |
() => catalogApplicationItems(rows, definition.fieldMap),
|
|
@@ -243,7 +255,8 @@ export const CatalogApplicationView = memo(function CatalogApplicationView({
|
|
| 243 |
key={definition.id}
|
| 244 |
definition={definition}
|
| 245 |
canEdit={editor.canEdit}
|
| 246 |
-
rows={editor.poolRows}
|
|
|
|
| 247 |
onOpen={onOpen}
|
| 248 |
/>
|
| 249 |
);
|
|
@@ -316,10 +329,11 @@ function catalogHashId(next?: string | null): string | null {
|
|
| 316 |
}
|
| 317 |
|
| 318 |
/** Child content for the shared AppFrame. Only the editor owns Back to library. */
|
| 319 |
-
export function CatalogLibrary({ definition, canEdit, rows, onOpen }: {
|
| 320 |
definition: CatalogApplicationDefinition;
|
| 321 |
canEdit: boolean;
|
| 322 |
rows: Row[];
|
|
|
|
| 323 |
onOpen: (pid: number) => void;
|
| 324 |
}) {
|
| 325 |
const [catalogs, setCatalogs] = useState<CatalogRecord[]>([]);
|
|
@@ -408,7 +422,7 @@ export function CatalogLibrary({ definition, canEdit, rows, onOpen }: {
|
|
| 408 |
};
|
| 409 |
|
| 410 |
if (openId) return <CatalogRecordOpener key={openId} appId={definition.id} id={openId}
|
| 411 |
-
definition={definition} canEdit={canEdit} rows={rows} onOpen={onOpen}
|
| 412 |
onBack={() => { setOpenId(null); catalogHashId(null); }} onSaved={replace} />;
|
| 413 |
|
| 414 |
const actionTitle = action?.kind === "create" ? "New catalog" : action?.kind === "rename" ? "Rename catalog" :
|
|
@@ -480,8 +494,9 @@ export function CatalogLibrary({ definition, canEdit, rows, onOpen }: {
|
|
| 480 |
</section>;
|
| 481 |
}
|
| 482 |
|
| 483 |
-
function CatalogRecordOpener({ appId, id, ...props }: {
|
| 484 |
appId: string; id: string; definition: CatalogApplicationDefinition; canEdit: boolean; rows: Row[];
|
|
|
|
| 485 |
onOpen: (pid: number) => void; onBack: () => void; onSaved: (catalog: CatalogRecord) => void;
|
| 486 |
}) {
|
| 487 |
const [record, setRecord] = useState<CatalogRecord | null>(null);
|
|
@@ -497,11 +512,16 @@ function CatalogRecordOpener({ appId, id, ...props }: {
|
|
| 497 |
});
|
| 498 |
return () => controller.abort();
|
| 499 |
}, [appId, id, attempt]);
|
| 500 |
-
|
|
|
|
| 501 |
return <section className="cg-catalog-record" aria-label="Catalog editor">
|
| 502 |
<button type="button" className="cg-btn" onClick={props.onBack}>Back to library</button>
|
| 503 |
{error ? <div className="cg-catalog-library-error" role="alert">{error}
|
| 504 |
<button type="button" className="cg-btn" onClick={() => setAttempt((value) => value + 1)}>Retry</button></div> :
|
|
|
|
|
|
|
|
|
|
|
|
|
| 505 |
<p role="status">Loading catalog</p>}
|
| 506 |
</section>;
|
| 507 |
}
|
|
|
|
| 221 |
onSave: (catalogs: CatalogSpec[], catalogSpec: CatalogViewSpec) => Promise<void>;
|
| 222 |
}
|
| 223 |
|
| 224 |
+
/** The library needs only the permission while rows load. The editor waits for the full editor data. */
|
| 225 |
+
export type CatalogPendingEditor = Pick<CatalogApplicationEditor, "canEdit">;
|
| 226 |
+
|
| 227 |
+
/** Rows the App has read so far, while its record walk is still running. */
|
| 228 |
+
export interface CatalogRowsPending {
|
| 229 |
+
read: number;
|
| 230 |
+
}
|
| 231 |
+
|
| 232 |
+
const NO_ROWS: Row[] = [];
|
| 233 |
+
|
| 234 |
/** Apps read C1 records. Sheets retain their read-only row projection. */
|
| 235 |
export const CatalogApplicationView = memo(function CatalogApplicationView({
|
| 236 |
definition,
|
| 237 |
rows,
|
| 238 |
onOpen,
|
| 239 |
editor,
|
| 240 |
+
rowsPending,
|
| 241 |
}: {
|
| 242 |
definition: CatalogApplicationDefinition;
|
| 243 |
rows: Row[];
|
| 244 |
onOpen: (pid: number) => void;
|
| 245 |
+
editor?: CatalogApplicationEditor | CatalogPendingEditor;
|
| 246 |
+
rowsPending?: CatalogRowsPending;
|
| 247 |
}) {
|
| 248 |
const items = useMemo(
|
| 249 |
() => catalogApplicationItems(rows, definition.fieldMap),
|
|
|
|
| 255 |
key={definition.id}
|
| 256 |
definition={definition}
|
| 257 |
canEdit={editor.canEdit}
|
| 258 |
+
rows={!rowsPending && "poolRows" in editor ? editor.poolRows : NO_ROWS}
|
| 259 |
+
rowsPending={rowsPending}
|
| 260 |
onOpen={onOpen}
|
| 261 |
/>
|
| 262 |
);
|
|
|
|
| 329 |
}
|
| 330 |
|
| 331 |
/** Child content for the shared AppFrame. Only the editor owns Back to library. */
|
| 332 |
+
export function CatalogLibrary({ definition, canEdit, rows, rowsPending, onOpen }: {
|
| 333 |
definition: CatalogApplicationDefinition;
|
| 334 |
canEdit: boolean;
|
| 335 |
rows: Row[];
|
| 336 |
+
rowsPending?: CatalogRowsPending;
|
| 337 |
onOpen: (pid: number) => void;
|
| 338 |
}) {
|
| 339 |
const [catalogs, setCatalogs] = useState<CatalogRecord[]>([]);
|
|
|
|
| 422 |
};
|
| 423 |
|
| 424 |
if (openId) return <CatalogRecordOpener key={openId} appId={definition.id} id={openId}
|
| 425 |
+
definition={definition} canEdit={canEdit} rows={rows} rowsPending={rowsPending} onOpen={onOpen}
|
| 426 |
onBack={() => { setOpenId(null); catalogHashId(null); }} onSaved={replace} />;
|
| 427 |
|
| 428 |
const actionTitle = action?.kind === "create" ? "New catalog" : action?.kind === "rename" ? "Rename catalog" :
|
|
|
|
| 494 |
</section>;
|
| 495 |
}
|
| 496 |
|
| 497 |
+
function CatalogRecordOpener({ appId, id, rowsPending, ...props }: {
|
| 498 |
appId: string; id: string; definition: CatalogApplicationDefinition; canEdit: boolean; rows: Row[];
|
| 499 |
+
rowsPending?: CatalogRowsPending;
|
| 500 |
onOpen: (pid: number) => void; onBack: () => void; onSaved: (catalog: CatalogRecord) => void;
|
| 501 |
}) {
|
| 502 |
const [record, setRecord] = useState<CatalogRecord | null>(null);
|
|
|
|
| 512 |
});
|
| 513 |
return () => controller.abort();
|
| 514 |
}, [appId, id, attempt]);
|
| 515 |
+
// The editor places records, so it mounts only once the App's record walk is complete.
|
| 516 |
+
if (record && !rowsPending) return <CatalogRecordEditor key={record.id} record={record} {...props} />;
|
| 517 |
return <section className="cg-catalog-record" aria-label="Catalog editor">
|
| 518 |
<button type="button" className="cg-btn" onClick={props.onBack}>Back to library</button>
|
| 519 |
{error ? <div className="cg-catalog-library-error" role="alert">{error}
|
| 520 |
<button type="button" className="cg-btn" onClick={() => setAttempt((value) => value + 1)}>Retry</button></div> :
|
| 521 |
+
rowsPending ? <div className="cg-app-launch-state" role="status" aria-label="Loading catalog records">
|
| 522 |
+
<span className="cg-app-launch-line is-wide" />
|
| 523 |
+
{rowsPending.read > 0 && <span>Loading records, {rowsPending.read.toLocaleString()} read</span>}
|
| 524 |
+
</div> :
|
| 525 |
<p role="status">Loading catalog</p>}
|
| 526 |
</section>;
|
| 527 |
}
|