Deploy AIOS web (React glide grid + FastAPI slice)
Browse files- RELEASES.json +1 -1
- VERSION +1 -1
- api/amazon_relational.py +20 -6
- api/registry_source.py +6 -4
- platform/core/received_changes.py +158 -57
- platform/core/shared_overlay.py +11 -0
- platform/modules/competitor_watch.py +22 -4
- platform/modules/product_data.py +8 -3
- web/public/AIOS_DEPLOY_RECEIPT +1 -1
RELEASES.json
CHANGED
|
@@ -1,4 +1,4 @@
|
|
| 1 |
{
|
| 2 |
-
"current": "
|
| 3 |
"releases": []
|
| 4 |
}
|
|
|
|
| 1 |
{
|
| 2 |
+
"current": "99ba051e73",
|
| 3 |
"releases": []
|
| 4 |
}
|
VERSION
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
|
|
|
|
| 1 |
+
99ba051e73
|
api/amazon_relational.py
CHANGED
|
@@ -872,6 +872,23 @@ def _missing_occurrence(source, partition, complete, predicate=''):
|
|
| 872 |
f'WHERE "_identity_rowid" = {source}.rowid)')
|
| 873 |
|
| 874 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 875 |
def _grid_identity(cur, key, names, prepared, is_view):
|
| 876 |
table = key.removeprefix(PREFIX)
|
| 877 |
identity_names = names | prepared
|
|
@@ -894,14 +911,13 @@ def _grid_identity(cur, key, names, prepared, is_view):
|
|
| 894 |
complete = " AND ".join(f"{value(name)} IS NOT NULL" for name in keys)
|
| 895 |
fallback = sorted(set((*keys, *_fallback_columns(table))))
|
| 896 |
partition = ", ".join(value(name, peer) for name in fallback)
|
|
|
|
| 897 |
if is_view:
|
| 898 |
if legacy:
|
| 899 |
partition += f', ({peer}."_connection_id" = {legacy})'
|
| 900 |
-
ranks.append(
|
| 901 |
-
f'ORDER BY {peer}."_amazon_publication_order") - 1 AS {_quoted(label)}')
|
| 902 |
occurrence = f'CAST({source}.{_quoted(label)} AS VARCHAR)'
|
| 903 |
else:
|
| 904 |
-
peer_complete = " AND ".join(f"{value(name, peer)} IS NOT NULL" for name in keys)
|
| 905 |
predicate = f'{peer}."_connection_id" = {legacy} AND ' if legacy else ''
|
| 906 |
occurrence = _missing_occurrence(source, partition, peer_complete, predicate)
|
| 907 |
if not legacy and amazon_schema.IDENTITY_OCCURRENCE in prepared:
|
|
@@ -925,9 +941,7 @@ def _grid_identity(cur, key, names, prepared, is_view):
|
|
| 925 |
ident = f'CASE WHEN {source}."_connection_id"={literal} THEN {old_ident} ELSE {ident} END'
|
| 926 |
spec = {'id': ident}
|
| 927 |
if ranks:
|
| 928 |
-
spec['from_sql'] = (
|
| 929 |
-
f'(SELECT *, row_number() OVER () - 1 AS "_amazon_publication_order" '
|
| 930 |
-
f'FROM {source}) AS {peer}) AS {source}')
|
| 931 |
if not is_view and table not in amazon_store.DERIVED_TABLES:
|
| 932 |
spec['page_key'] = f'{source}.rowid'
|
| 933 |
return spec
|
|
|
|
| 872 |
f'WHERE "_identity_rowid" = {source}.rowid)')
|
| 873 |
|
| 874 |
|
| 875 |
+
def _occurrence_view(source, peer, ranks):
|
| 876 |
+
"""Number equal-key occurrences over incomplete-key rows only, in publication order.
|
| 877 |
+
|
| 878 |
+
A complete key never shares a partition with an incomplete one, so the narrow window over
|
| 879 |
+
incomplete rows yields the same occurrence a whole-table window would.
|
| 880 |
+
"""
|
| 881 |
+
order = '"_amazon_publication_order"'
|
| 882 |
+
numbered = f'(SELECT *, row_number() OVER () - 1 AS {order} FROM {source})'
|
| 883 |
+
joins = "".join(
|
| 884 |
+
f' LEFT JOIN (SELECT {order}, row_number() OVER (PARTITION BY {partition} '
|
| 885 |
+
f'ORDER BY {peer}.{order}) - 1 AS {_quoted(label)} FROM {numbered} AS {peer} '
|
| 886 |
+
f'WHERE NOT ({complete})) AS "_amazon_rank_{index}" USING ({order})'
|
| 887 |
+
for index, (label, partition, complete) in enumerate(ranks))
|
| 888 |
+
labels = ", ".join(f'"_amazon_rank_{index}".{_quoted(label)}' for index, (label, _p, _c) in enumerate(ranks))
|
| 889 |
+
return f'(SELECT {peer}.*, {labels} FROM {numbered} AS {peer}{joins}) AS {source}'
|
| 890 |
+
|
| 891 |
+
|
| 892 |
def _grid_identity(cur, key, names, prepared, is_view):
|
| 893 |
table = key.removeprefix(PREFIX)
|
| 894 |
identity_names = names | prepared
|
|
|
|
| 911 |
complete = " AND ".join(f"{value(name)} IS NOT NULL" for name in keys)
|
| 912 |
fallback = sorted(set((*keys, *_fallback_columns(table))))
|
| 913 |
partition = ", ".join(value(name, peer) for name in fallback)
|
| 914 |
+
peer_complete = " AND ".join(f"{value(name, peer)} IS NOT NULL" for name in keys)
|
| 915 |
if is_view:
|
| 916 |
if legacy:
|
| 917 |
partition += f', ({peer}."_connection_id" = {legacy})'
|
| 918 |
+
ranks.append((label, partition, peer_complete))
|
|
|
|
| 919 |
occurrence = f'CAST({source}.{_quoted(label)} AS VARCHAR)'
|
| 920 |
else:
|
|
|
|
| 921 |
predicate = f'{peer}."_connection_id" = {legacy} AND ' if legacy else ''
|
| 922 |
occurrence = _missing_occurrence(source, partition, peer_complete, predicate)
|
| 923 |
if not legacy and amazon_schema.IDENTITY_OCCURRENCE in prepared:
|
|
|
|
| 941 |
ident = f'CASE WHEN {source}."_connection_id"={literal} THEN {old_ident} ELSE {ident} END'
|
| 942 |
spec = {'id': ident}
|
| 943 |
if ranks:
|
| 944 |
+
spec['from_sql'] = _occurrence_view(source, peer, ranks)
|
|
|
|
|
|
|
| 945 |
if not is_view and table not in amazon_store.DERIVED_TABLES:
|
| 946 |
spec['page_key'] = f'{source}.rowid'
|
| 947 |
return spec
|
api/registry_source.py
CHANGED
|
@@ -558,10 +558,13 @@ class CanonicalStore(BatchStore):
|
|
| 558 |
[self.uname, key, 'members'], self.row_ids, source_limit=cohort.MAX_MEMBERS * 2))
|
| 559 |
for key, meta in definitions.items() if isinstance(meta, dict)}}
|
| 560 |
elif name == 'loopable_route_system':
|
| 561 |
-
|
|
|
|
| 562 |
if meta_key not in self.metadata or fresh:
|
| 563 |
base = object_values(self._runtime, name, omit=('routes', 'zones'))
|
| 564 |
-
|
|
|
|
|
|
|
| 565 |
['routes'], drop=('stops',)) for index, route in batch]
|
| 566 |
base['zones'] = [meta for batch in array_values(self._runtime, name, ['zones'], drop=(
|
| 567 |
'memberPids', 'member_pids', 'memberships', 'legacyMembershipEvidence')) for _, meta in batch]
|
|
@@ -571,8 +574,7 @@ class CanonicalStore(BatchStore):
|
|
| 571 |
value['routes'], value['zones'] = [], []
|
| 572 |
for index, route in saved['routes']:
|
| 573 |
if isinstance(route, dict):
|
| 574 |
-
|
| 575 |
-
route = dict(route, stops=[] if self.table_key == 'product_data' else self._selected_json(
|
| 576 |
name, ['routes', index, 'stops'], array=True, id_key='customerPid'))
|
| 577 |
value['routes'].append(route)
|
| 578 |
value['zones'] = copy.deepcopy(saved['zones'])
|
|
|
|
| 558 |
[self.uname, key, 'members'], self.row_ids, source_limit=cohort.MAX_MEMBERS * 2))
|
| 559 |
for key, meta in definitions.items() if isinstance(meta, dict)}}
|
| 560 |
elif name == 'loopable_route_system':
|
| 561 |
+
# The product shape omits routes, so the cached definitions are keyed by that shape.
|
| 562 |
+
meta_key = ('route_definitions', name, self.table_key == 'product_data')
|
| 563 |
if meta_key not in self.metadata or fresh:
|
| 564 |
base = object_values(self._runtime, name, omit=('routes', 'zones'))
|
| 565 |
+
# A Route stops at customers, so a product batch reads only the Zone definitions.
|
| 566 |
+
base['routes'] = [] if self.table_key == 'product_data' else [
|
| 567 |
+
(index, route) for batch in array_values(self._runtime, name,
|
| 568 |
['routes'], drop=('stops',)) for index, route in batch]
|
| 569 |
base['zones'] = [meta for batch in array_values(self._runtime, name, ['zones'], drop=(
|
| 570 |
'memberPids', 'member_pids', 'memberships', 'legacyMembershipEvidence')) for _, meta in batch]
|
|
|
|
| 574 |
value['routes'], value['zones'] = [], []
|
| 575 |
for index, route in saved['routes']:
|
| 576 |
if isinstance(route, dict):
|
| 577 |
+
route = dict(route, stops=self._selected_json(
|
|
|
|
| 578 |
name, ['routes', index, 'stops'], array=True, id_key='customerPid'))
|
| 579 |
value['routes'].append(route)
|
| 580 |
value['zones'] = copy.deepcopy(saved['zones'])
|
platform/core/received_changes.py
CHANGED
|
@@ -20,6 +20,8 @@ JOURNAL_DEADLINE_SECONDS = 120
|
|
| 20 |
BASELINE_LOCK_SECONDS = 300
|
| 21 |
BASELINE_CHUNK_BYTES = 8 * 1024 * 1024
|
| 22 |
PROJECTION_VERSION = 1
|
|
|
|
|
|
|
| 23 |
_LISTENERS = []
|
| 24 |
_PROJECTIONS = {}
|
| 25 |
_LOCK = threading.RLock()
|
|
@@ -293,9 +295,8 @@ def _blob_path(identity, kind, digest):
|
|
| 293 |
return f"{STORE_KEY}/observations/{identity}/{kind}/{digest}"
|
| 294 |
|
| 295 |
|
| 296 |
-
def
|
| 297 |
from core import blob_envelope as be
|
| 298 |
-
raw = rt.download_bytes(path)
|
| 299 |
if raw is None or len(raw) > BASELINE_CHUNK_BYTES:
|
| 300 |
raise BaselineUnavailable("A required observation blob is missing or oversized.")
|
| 301 |
data = be.open(rt.key, path, raw) if be.is_sealed(raw) else bytes(raw)
|
|
@@ -304,16 +305,22 @@ def _read_blob(rt, path, digest, size):
|
|
| 304 |
return data
|
| 305 |
|
| 306 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 307 |
def _write_blob(rt, path, data):
|
| 308 |
"""Reuse immutable content and verify persistence before publishing its reference."""
|
| 309 |
from core import blob_envelope as be
|
| 310 |
digest, size = _digest(data), len(data)
|
| 311 |
-
|
|
|
|
| 312 |
stored = be.seal(rt.key, path, data)
|
| 313 |
if len(stored) > BASELINE_CHUNK_BYTES:
|
| 314 |
raise BaselineUnavailable("The observation blob exceeds the store chunk size.")
|
| 315 |
rt.upload_bytes(path, stored)
|
| 316 |
-
|
|
|
|
| 317 |
return {"sha256": digest, "bytes": size}
|
| 318 |
|
| 319 |
|
|
@@ -331,7 +338,19 @@ def _persist_baseline(rt, identity, binding, path, snapshot, stamp):
|
|
| 331 |
return _write_blob(rt, _blob_path(identity, "manifests", _digest(data)), data)
|
| 332 |
|
| 333 |
|
| 334 |
-
def
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 335 |
try:
|
| 336 |
data = _read_blob(rt, _blob_path(identity, "manifests", reference["sha256"]),
|
| 337 |
reference["sha256"], reference["bytes"])
|
|
@@ -342,15 +361,18 @@ def _restore_baseline(rt, identity, binding, reference, path, con):
|
|
| 342 |
or not manifest.get("at")):
|
| 343 |
raise BaselineUnavailable("The observation manifest has a different identity or shape.")
|
| 344 |
_at(manifest["at"])
|
| 345 |
-
|
| 346 |
-
|
| 347 |
-
|
| 348 |
-
|
| 349 |
-
|
| 350 |
-
|
| 351 |
-
|
| 352 |
-
|
| 353 |
-
|
|
|
|
|
|
|
|
|
|
| 354 |
raise BaselineUnavailable("The observation snapshot failed content verification.")
|
| 355 |
_validate_projection(con, path)
|
| 356 |
return manifest
|
|
@@ -360,17 +382,52 @@ def _restore_baseline(rt, identity, binding, reference, path, con):
|
|
| 360 |
raise BaselineUnavailable("The required observation could not be verified.") from exc
|
| 361 |
|
| 362 |
|
| 363 |
-
def
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 364 |
schema = con.execute("DESCRIBE SELECT * FROM read_parquet(?)", [str(path)]).fetchall()
|
| 365 |
if [(row[0], row[1]) for row in schema] != [("row_id", "VARCHAR"), ("image", "VARCHAR")]:
|
| 366 |
raise BaselineUnavailable("The observation projection has an invalid schema.")
|
| 367 |
-
|
| 368 |
-
|
| 369 |
-
|
| 370 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 371 |
raise BaselineUnavailable("A received-change projection requires unique non-null row IDs and images.")
|
| 372 |
|
| 373 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 374 |
def _observation_state(rt, identity):
|
| 375 |
document = _document(rt, rt.get(STORE_KEY, fresh=True))
|
| 376 |
states = document.get("observations", {})
|
|
@@ -392,7 +449,7 @@ def _observation_state(rt, identity):
|
|
| 392 |
return state, document
|
| 393 |
|
| 394 |
|
| 395 |
-
def _move_baseline(rt, identity, expected, updated):
|
| 396 |
"""Serialize replica transitions in the existing tenant journal's update boundary."""
|
| 397 |
def move(raw):
|
| 398 |
document = _document(rt, raw)
|
|
@@ -400,6 +457,8 @@ def _move_baseline(rt, identity, expected, updated):
|
|
| 400 |
if states.get(identity) != expected:
|
| 401 |
raise BaselineUnavailable("The observation changed on another replica. Retry publication.")
|
| 402 |
states[identity] = updated
|
|
|
|
|
|
|
| 403 |
return document
|
| 404 |
result = rt.update(STORE_KEY, move, flush="sync")
|
| 405 |
if result is None or result is False:
|
|
@@ -422,19 +481,24 @@ def _query_watchdog(con, deadline):
|
|
| 422 |
|
| 423 |
|
| 424 |
def _compare_baseline(rt, source, database, spec, con, baseline, current, snapshot, deadline=None):
|
| 425 |
-
|
| 426 |
-
|
| 427 |
-
|
|
|
|
|
|
|
| 428 |
# A diff wider than the retained window cannot be replayed, so it is recorded as one bulk change.
|
| 429 |
-
if
|
|
|
|
| 430 |
print(f"[received-changes] {database}: {count} changed rows exceed the journal's "
|
| 431 |
f"{MAX_EVENTS}-event window; recorded as a bulk change, rows not journaled", flush=True)
|
| 432 |
return
|
| 433 |
-
cursor, done = con.cursor(), 0
|
| 434 |
try:
|
| 435 |
with _query_watchdog(cursor, deadline):
|
| 436 |
-
|
| 437 |
-
|
|
|
|
|
|
|
| 438 |
while batch := cursor.fetchmany(256):
|
| 439 |
for row_id, before_json, after_json in batch:
|
| 440 |
# Stop and commit: a rerun cannot resume, since each journaled receipt still costs a store write.
|
|
@@ -459,13 +523,17 @@ def _compare_baseline(rt, source, database, spec, con, baseline, current, snapsh
|
|
| 459 |
cursor.close()
|
| 460 |
|
| 461 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 462 |
def _snapshot(rt, source, database, spec, con, root, manifest, deadline=None):
|
| 463 |
"""Compare verified durable observations in batches, recovering interrupted comparisons first."""
|
| 464 |
if not spec.get("cols"):
|
| 465 |
return
|
| 466 |
if _uncommitted():
|
| 467 |
raise BaselineUnavailable("Connector observations require a completed publication transaction.")
|
| 468 |
-
identity =
|
| 469 |
binding = {"version": PROJECTION_VERSION, "tenant": _tenant(rt), "source": source,
|
| 470 |
"database": database, "projection": _projection_shape(spec)}
|
| 471 |
snapshot, stamp = str(manifest["snapshot_id"]), manifest.get("completed_at")
|
|
@@ -477,23 +545,28 @@ def _snapshot(rt, source, database, spec, con, root, manifest, deadline=None):
|
|
| 477 |
or any(event["source"] == source and event["database"] == database
|
| 478 |
for event in document["events"])):
|
| 479 |
raise BaselineUnavailable("A legacy observation requires verified migration before comparison.")
|
|
|
|
| 480 |
with tempfile.TemporaryDirectory(dir=root, prefix="projection-") as directory:
|
| 481 |
baseline, current = Path(directory) / "baseline.parquet", Path(directory) / "current.parquet"
|
| 482 |
|
| 483 |
-
def complete(pending):
|
| 484 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 485 |
if pending["current"]:
|
| 486 |
-
|
| 487 |
-
_compare_baseline(rt, source, database, spec, con, baseline, current, observed["snapshot"],
|
| 488 |
-
deadline)
|
| 489 |
committed = {"current": pending["pending"], "pending": None}
|
| 490 |
-
_move_baseline(rt, identity, pending, committed)
|
|
|
|
| 491 |
return committed
|
| 492 |
|
| 493 |
if state and state["pending"]:
|
| 494 |
state = complete(state)
|
| 495 |
if state:
|
| 496 |
-
prior = _restore_baseline(rt, identity, binding, state["current"], baseline, con)
|
| 497 |
prior_at = dt.datetime.fromisoformat(prior["at"].replace("Z", "+00:00"))
|
| 498 |
incoming_at = dt.datetime.fromisoformat(stamp.replace("Z", "+00:00"))
|
| 499 |
if prior["snapshot"] == snapshot or incoming_at < prior_at:
|
|
@@ -506,12 +579,41 @@ def _snapshot(rt, source, database, spec, con, root, manifest, deadline=None):
|
|
| 506 |
f"CAST(to_json(struct_pack({expressions})) AS VARCHAR) AS image "
|
| 507 |
f"FROM {spec.get('from_sql') or spec['table']}{where}")
|
| 508 |
current.unlink(missing_ok=True)
|
| 509 |
-
|
| 510 |
-
_validate_projection(con, current)
|
| 511 |
reference = _persist_baseline(rt, identity, binding, current, snapshot, stamp)
|
| 512 |
pending = {"current": state["current"] if state else None, "pending": reference}
|
| 513 |
_move_baseline(rt, identity, state, pending)
|
| 514 |
-
complete(pending)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 515 |
|
| 516 |
|
| 517 |
def mirror_hook(connector, tenant_key, manifest, _state_rows, paths):
|
|
@@ -539,25 +641,24 @@ def mirror_hook(connector, tenant_key, manifest, _state_rows, paths):
|
|
| 539 |
started = time.monotonic()
|
| 540 |
specs = resolver(rt, con)
|
| 541 |
resolution_seconds = time.monotonic() - started
|
| 542 |
-
|
| 543 |
-
|
| 544 |
-
|
| 545 |
-
|
| 546 |
-
|
| 547 |
-
|
| 548 |
-
|
| 549 |
-
|
| 550 |
-
|
| 551 |
-
|
| 552 |
-
|
| 553 |
-
|
| 554 |
-
|
| 555 |
-
|
| 556 |
-
|
| 557 |
-
|
| 558 |
-
|
| 559 |
-
|
| 560 |
-
flush=True)
|
| 561 |
finally:
|
| 562 |
con.close()
|
| 563 |
|
|
|
|
| 20 |
BASELINE_LOCK_SECONDS = 300
|
| 21 |
BASELINE_CHUNK_BYTES = 8 * 1024 * 1024
|
| 22 |
PROJECTION_VERSION = 1
|
| 23 |
+
# A 128-bit digest of each image: the compare join carries it, never the image itself.
|
| 24 |
+
IMAGE_DIGEST = "md5_number(image)"
|
| 25 |
_LISTENERS = []
|
| 26 |
_PROJECTIONS = {}
|
| 27 |
_LOCK = threading.RLock()
|
|
|
|
| 295 |
return f"{STORE_KEY}/observations/{identity}/{kind}/{digest}"
|
| 296 |
|
| 297 |
|
| 298 |
+
def _open_blob(rt, path, raw, digest, size):
|
| 299 |
from core import blob_envelope as be
|
|
|
|
| 300 |
if raw is None or len(raw) > BASELINE_CHUNK_BYTES:
|
| 301 |
raise BaselineUnavailable("A required observation blob is missing or oversized.")
|
| 302 |
data = be.open(rt.key, path, raw) if be.is_sealed(raw) else bytes(raw)
|
|
|
|
| 305 |
return data
|
| 306 |
|
| 307 |
|
| 308 |
+
def _read_blob(rt, path, digest, size):
|
| 309 |
+
return _open_blob(rt, path, rt.download_bytes(path), digest, size)
|
| 310 |
+
|
| 311 |
+
|
| 312 |
def _write_blob(rt, path, data):
|
| 313 |
"""Reuse immutable content and verify persistence before publishing its reference."""
|
| 314 |
from core import blob_envelope as be
|
| 315 |
digest, size = _digest(data), len(data)
|
| 316 |
+
raw = rt.download_bytes(path)
|
| 317 |
+
if raw is None:
|
| 318 |
stored = be.seal(rt.key, path, data)
|
| 319 |
if len(stored) > BASELINE_CHUNK_BYTES:
|
| 320 |
raise BaselineUnavailable("The observation blob exceeds the store chunk size.")
|
| 321 |
rt.upload_bytes(path, stored)
|
| 322 |
+
raw = rt.download_bytes(path)
|
| 323 |
+
_open_blob(rt, path, raw, digest, size)
|
| 324 |
return {"sha256": digest, "bytes": size}
|
| 325 |
|
| 326 |
|
|
|
|
| 338 |
return _write_blob(rt, _blob_path(identity, "manifests", _digest(data)), data)
|
| 339 |
|
| 340 |
|
| 341 |
+
def _cached_baseline(cache, identity, reference):
|
| 342 |
+
return cache / f"{identity}-{reference['sha256']}.parquet"
|
| 343 |
+
|
| 344 |
+
|
| 345 |
+
def _cache_baseline(cache, identity, reference, path):
|
| 346 |
+
"""Keep the committed observation on local disk so the next pass skips its download."""
|
| 347 |
+
cache.mkdir(parents=True, exist_ok=True)
|
| 348 |
+
for stale in cache.glob(identity + "-*.parquet"):
|
| 349 |
+
stale.unlink(missing_ok=True)
|
| 350 |
+
os.replace(path, _cached_baseline(cache, identity, reference))
|
| 351 |
+
|
| 352 |
+
|
| 353 |
+
def _restore_baseline(rt, identity, binding, reference, path, con, cache):
|
| 354 |
try:
|
| 355 |
data = _read_blob(rt, _blob_path(identity, "manifests", reference["sha256"]),
|
| 356 |
reference["sha256"], reference["bytes"])
|
|
|
|
| 361 |
or not manifest.get("at")):
|
| 362 |
raise BaselineUnavailable("The observation manifest has a different identity or shape.")
|
| 363 |
_at(manifest["at"])
|
| 364 |
+
cached = _cached_baseline(cache, identity, reference)
|
| 365 |
+
if cached.exists():
|
| 366 |
+
with cached.open("rb") as source:
|
| 367 |
+
chunks = iter(lambda: source.read(BASELINE_CHUNK_BYTES), b"")
|
| 368 |
+
content, size = _write_chunks(path, chunks)
|
| 369 |
+
if content != manifest["sha256"] or size != manifest["bytes"]:
|
| 370 |
+
cached.unlink(missing_ok=True)
|
| 371 |
+
if not cached.exists():
|
| 372 |
+
content, size = _write_chunks(path, (
|
| 373 |
+
_read_blob(rt, _blob_path(identity, "chunks", part["sha256"]), part["sha256"], part["bytes"])
|
| 374 |
+
for part in manifest["parts"]))
|
| 375 |
+
if content != manifest["sha256"] or size != manifest["bytes"]:
|
| 376 |
raise BaselineUnavailable("The observation snapshot failed content verification.")
|
| 377 |
_validate_projection(con, path)
|
| 378 |
return manifest
|
|
|
|
| 382 |
raise BaselineUnavailable("The required observation could not be verified.") from exc
|
| 383 |
|
| 384 |
|
| 385 |
+
def _write_chunks(path, chunks):
|
| 386 |
+
content, size = hashlib.sha256(), 0
|
| 387 |
+
with path.open("wb") as handle:
|
| 388 |
+
for chunk in chunks:
|
| 389 |
+
handle.write(chunk)
|
| 390 |
+
content.update(chunk)
|
| 391 |
+
size += len(chunk)
|
| 392 |
+
return content.hexdigest(), size
|
| 393 |
+
|
| 394 |
+
|
| 395 |
+
def _projection_issues(con, path):
|
| 396 |
+
"""Return (shared row IDs, rows sharing them, any null) for a projection of valid schema."""
|
| 397 |
schema = con.execute("DESCRIBE SELECT * FROM read_parquet(?)", [str(path)]).fetchall()
|
| 398 |
if [(row[0], row[1]) for row in schema] != [("row_id", "VARCHAR"), ("image", "VARCHAR")]:
|
| 399 |
raise BaselineUnavailable("The observation projection has an invalid schema.")
|
| 400 |
+
return con.execute("SELECT count(*) FILTER (WHERE n > 1), coalesce(sum(n) FILTER (WHERE n > 1), 0), "
|
| 401 |
+
"coalesce(bool_or(row_id IS NULL OR missing > 0), false) FROM (SELECT row_id, "
|
| 402 |
+
"count(*) AS n, count(*) - count(image) AS missing FROM read_parquet(?) GROUP BY row_id)",
|
| 403 |
+
[str(path)]).fetchone()
|
| 404 |
+
|
| 405 |
+
|
| 406 |
+
def _validate_projection(con, path):
|
| 407 |
+
shared, _rows, missing = _projection_issues(con, path)
|
| 408 |
+
if shared or missing:
|
| 409 |
raise BaselineUnavailable("A received-change projection requires unique non-null row IDs and images.")
|
| 410 |
|
| 411 |
|
| 412 |
+
def _project(con, query, path, database):
|
| 413 |
+
"""Write the projection, leaving out row IDs the connector identity gives to several rows."""
|
| 414 |
+
raw = path.with_name("projection.parquet")
|
| 415 |
+
con.execute(f"COPY ({query}) TO ? (FORMAT PARQUET, COMPRESSION ZSTD)", [str(raw)])
|
| 416 |
+
shared, rows, missing = _projection_issues(con, raw)
|
| 417 |
+
if missing:
|
| 418 |
+
raise BaselineUnavailable("A received-change projection requires non-null row IDs and images.")
|
| 419 |
+
if not shared:
|
| 420 |
+
os.replace(raw, path)
|
| 421 |
+
return
|
| 422 |
+
# An ID shared by several rows names no single record, so it cannot carry a change event.
|
| 423 |
+
print(f"[received-changes] {database}: {rows} rows share {shared} row IDs; "
|
| 424 |
+
"those IDs are not observed until the connector identity separates them", flush=True)
|
| 425 |
+
con.execute("COPY (SELECT * FROM read_parquet($1) WHERE row_id NOT IN (SELECT row_id FROM read_parquet($1) "
|
| 426 |
+
"GROUP BY row_id HAVING count(*) > 1)) TO $2 (FORMAT PARQUET, COMPRESSION ZSTD)",
|
| 427 |
+
[str(raw), str(path)])
|
| 428 |
+
raw.unlink()
|
| 429 |
+
|
| 430 |
+
|
| 431 |
def _observation_state(rt, identity):
|
| 432 |
document = _document(rt, rt.get(STORE_KEY, fresh=True))
|
| 433 |
states = document.get("observations", {})
|
|
|
|
| 449 |
return state, document
|
| 450 |
|
| 451 |
|
| 452 |
+
def _move_baseline(rt, identity, expected, updated, observed_at=None):
|
| 453 |
"""Serialize replica transitions in the existing tenant journal's update boundary."""
|
| 454 |
def move(raw):
|
| 455 |
document = _document(rt, raw)
|
|
|
|
| 457 |
if states.get(identity) != expected:
|
| 458 |
raise BaselineUnavailable("The observation changed on another replica. Retry publication.")
|
| 459 |
states[identity] = updated
|
| 460 |
+
if observed_at:
|
| 461 |
+
document.setdefault("observed", {})[identity] = observed_at
|
| 462 |
return document
|
| 463 |
result = rt.update(STORE_KEY, move, flush="sync")
|
| 464 |
if result is None or result is False:
|
|
|
|
| 481 |
|
| 482 |
|
| 483 |
def _compare_baseline(rt, source, database, spec, con, baseline, current, snapshot, deadline=None):
|
| 484 |
+
digests = f"(SELECT row_id, {IMAGE_DIGEST} AS image FROM read_parquet({{}}))"
|
| 485 |
+
changed = (f"SELECT coalesce(b.row_id, a.row_id) FROM {digests.format('$1')} b FULL OUTER JOIN "
|
| 486 |
+
f"{digests.format('$2')} a USING (row_id) WHERE b.image IS DISTINCT FROM a.image")
|
| 487 |
+
files = [str(baseline), str(current)]
|
| 488 |
+
ids = [row[0] for row in con.execute(changed + f" LIMIT {MAX_EVENTS + 1}", files).fetchall()]
|
| 489 |
# A diff wider than the retained window cannot be replayed, so it is recorded as one bulk change.
|
| 490 |
+
if len(ids) > MAX_EVENTS:
|
| 491 |
+
count = con.execute(f"SELECT count(*) FROM ({changed})", files).fetchone()[0]
|
| 492 |
print(f"[received-changes] {database}: {count} changed rows exceed the journal's "
|
| 493 |
f"{MAX_EVENTS}-event window; recorded as a bulk change, rows not journaled", flush=True)
|
| 494 |
return
|
| 495 |
+
count, cursor, done = len(ids), con.cursor(), 0
|
| 496 |
try:
|
| 497 |
with _query_watchdog(cursor, deadline):
|
| 498 |
+
picked = "(SELECT * FROM read_parquet({}) WHERE row_id IN (SELECT unnest($3::VARCHAR[])))"
|
| 499 |
+
cursor.execute(f"SELECT coalesce(b.row_id, a.row_id), b.image, a.image FROM {picked.format('$1')} b "
|
| 500 |
+
f"FULL OUTER JOIN {picked.format('$2')} a USING (row_id) "
|
| 501 |
+
"ORDER BY coalesce(b.row_id, a.row_id)", [*files, ids])
|
| 502 |
while batch := cursor.fetchmany(256):
|
| 503 |
for row_id, before_json, after_json in batch:
|
| 504 |
# Stop and commit: a rerun cannot resume, since each journaled receipt still costs a store write.
|
|
|
|
| 523 |
cursor.close()
|
| 524 |
|
| 525 |
|
| 526 |
+
def _observation_identity(rt, source, database):
|
| 527 |
+
return hashlib.sha256(json.dumps([_tenant(rt), source, database]).encode()).hexdigest()
|
| 528 |
+
|
| 529 |
+
|
| 530 |
def _snapshot(rt, source, database, spec, con, root, manifest, deadline=None):
|
| 531 |
"""Compare verified durable observations in batches, recovering interrupted comparisons first."""
|
| 532 |
if not spec.get("cols"):
|
| 533 |
return
|
| 534 |
if _uncommitted():
|
| 535 |
raise BaselineUnavailable("Connector observations require a completed publication transaction.")
|
| 536 |
+
identity = _observation_identity(rt, source, database)
|
| 537 |
binding = {"version": PROJECTION_VERSION, "tenant": _tenant(rt), "source": source,
|
| 538 |
"database": database, "projection": _projection_shape(spec)}
|
| 539 |
snapshot, stamp = str(manifest["snapshot_id"]), manifest.get("completed_at")
|
|
|
|
| 545 |
or any(event["source"] == source and event["database"] == database
|
| 546 |
for event in document["events"])):
|
| 547 |
raise BaselineUnavailable("A legacy observation requires verified migration before comparison.")
|
| 548 |
+
cache = root / "baselines"
|
| 549 |
with tempfile.TemporaryDirectory(dir=root, prefix="projection-") as directory:
|
| 550 |
baseline, current = Path(directory) / "baseline.parquet", Path(directory) / "current.parquet"
|
| 551 |
|
| 552 |
+
def complete(pending, observed=None):
|
| 553 |
+
# A pending reference this pass wrote is already on disk; one from an earlier pass is restored.
|
| 554 |
+
if observed is None:
|
| 555 |
+
observed = _restore_baseline(rt, identity, binding, pending["pending"], current, con,
|
| 556 |
+
cache)["snapshot"]
|
| 557 |
+
if pending["current"]:
|
| 558 |
+
_restore_baseline(rt, identity, binding, pending["current"], baseline, con, cache)
|
| 559 |
if pending["current"]:
|
| 560 |
+
_compare_baseline(rt, source, database, spec, con, baseline, current, observed, deadline)
|
|
|
|
|
|
|
| 561 |
committed = {"current": pending["pending"], "pending": None}
|
| 562 |
+
_move_baseline(rt, identity, pending, committed, observed_at=_at())
|
| 563 |
+
_cache_baseline(cache, identity, pending["pending"], current)
|
| 564 |
return committed
|
| 565 |
|
| 566 |
if state and state["pending"]:
|
| 567 |
state = complete(state)
|
| 568 |
if state:
|
| 569 |
+
prior = _restore_baseline(rt, identity, binding, state["current"], baseline, con, cache)
|
| 570 |
prior_at = dt.datetime.fromisoformat(prior["at"].replace("Z", "+00:00"))
|
| 571 |
incoming_at = dt.datetime.fromisoformat(stamp.replace("Z", "+00:00"))
|
| 572 |
if prior["snapshot"] == snapshot or incoming_at < prior_at:
|
|
|
|
| 579 |
f"CAST(to_json(struct_pack({expressions})) AS VARCHAR) AS image "
|
| 580 |
f"FROM {spec.get('from_sql') or spec['table']}{where}")
|
| 581 |
current.unlink(missing_ok=True)
|
| 582 |
+
_project(con, query, current, database)
|
|
|
|
| 583 |
reference = _persist_baseline(rt, identity, binding, current, snapshot, stamp)
|
| 584 |
pending = {"current": state["current"] if state else None, "pending": reference}
|
| 585 |
_move_baseline(rt, identity, state, pending)
|
| 586 |
+
complete(pending, snapshot)
|
| 587 |
+
|
| 588 |
+
|
| 589 |
+
def _publication_order(rt, connector, specs, paths):
|
| 590 |
+
"""Least recently observed databases first, then the smallest, so a deadline defers in rotation."""
|
| 591 |
+
observed = _document(rt, rt.get(STORE_KEY, fresh=True)).get("observed") or {}
|
| 592 |
+
|
| 593 |
+
def priority(item):
|
| 594 |
+
database, spec = item
|
| 595 |
+
files = paths.get(spec.get("table"), ()) if isinstance(spec, dict) else ()
|
| 596 |
+
return (str(observed.get(_observation_identity(rt, connector, str(database))) or ""),
|
| 597 |
+
sum(os.path.getsize(path) for path in files if os.path.exists(path)))
|
| 598 |
+
return sorted(specs.items(), key=priority)
|
| 599 |
+
|
| 600 |
+
|
| 601 |
+
def _project_database(rt, connector, database, spec, con, root, manifest, deadline):
|
| 602 |
+
"""Return None when the database committed, else the named reason it was deferred."""
|
| 603 |
+
import duckdb
|
| 604 |
+
from core import request_timing
|
| 605 |
+
try:
|
| 606 |
+
with _query_watchdog(con, deadline):
|
| 607 |
+
_snapshot(rt, connector, database, spec, con, root, manifest, deadline)
|
| 608 |
+
except request_timing.RequestDeadlineExceeded as exc:
|
| 609 |
+
# The watchdog raises on exit after a clean body too; no cause means the database committed.
|
| 610 |
+
return None if exc.__cause__ is None else "journal deadline passed"
|
| 611 |
+
except BaselineUnavailable as exc:
|
| 612 |
+
return f"BaselineUnavailable: {exc}"
|
| 613 |
+
except duckdb.Error as exc:
|
| 614 |
+
# One database's failed query defers it alone; its stored baseline stays for the next pass.
|
| 615 |
+
return type(exc).__name__
|
| 616 |
+
return None
|
| 617 |
|
| 618 |
|
| 619 |
def mirror_hook(connector, tenant_key, manifest, _state_rows, paths):
|
|
|
|
| 641 |
started = time.monotonic()
|
| 642 |
specs = resolver(rt, con)
|
| 643 |
resolution_seconds = time.monotonic() - started
|
| 644 |
+
snapshots_started = time.monotonic()
|
| 645 |
+
completed = deferred = skipped = 0
|
| 646 |
+
for database, spec in _publication_order(rt, connector, specs, paths):
|
| 647 |
+
# A skipped database keeps its stored baseline and leads the next publication.
|
| 648 |
+
if time.monotonic() >= deadline[0]:
|
| 649 |
+
skipped += 1
|
| 650 |
+
print(f"[received-changes] {database}: skipped, journal deadline passed", flush=True)
|
| 651 |
+
continue
|
| 652 |
+
reason = _project_database(rt, connector, str(database), spec, con, root, manifest, deadline)
|
| 653 |
+
if reason is None:
|
| 654 |
+
completed += 1
|
| 655 |
+
continue
|
| 656 |
+
deferred += 1
|
| 657 |
+
_LOG.warning("Received-change projection deferred for %s: %s", database, reason)
|
| 658 |
+
print(f'[received-changes] publication connector={connector} tables={len(specs)} '
|
| 659 |
+
f'completed={completed} deferred={deferred} skipped={skipped} '
|
| 660 |
+
f'resolution={resolution_seconds:.3f}s snapshots={time.monotonic() - snapshots_started:.3f}s',
|
| 661 |
+
flush=True)
|
|
|
|
| 662 |
finally:
|
| 663 |
con.close()
|
| 664 |
|
platform/core/shared_overlay.py
CHANGED
|
@@ -861,6 +861,15 @@ def clear_row(table_key, pid, st=None):
|
|
| 861 |
return hit[0]
|
| 862 |
|
| 863 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 864 |
def prune(table_key, live_pids, st=None, *, pass_id=None, validate_source=None):
|
| 865 |
"""Drop shared cells for rows that no longer exist. Returns how many rows were dropped.
|
| 866 |
|
|
@@ -878,6 +887,8 @@ def prune(table_key, live_pids, st=None, *, pass_id=None, validate_source=None):
|
|
| 878 |
keep = {_pid(p) for p in live_pids}
|
| 879 |
if not keep:
|
| 880 |
return 0
|
|
|
|
|
|
|
| 881 |
dropped = []
|
| 882 |
completed = False
|
| 883 |
|
|
|
|
| 861 |
return hit[0]
|
| 862 |
|
| 863 |
|
| 864 |
+
def _stored_prune_pass(table_key, st):
|
| 865 |
+
"""The committed prune marker read by path on a native store, else None (the write decides)."""
|
| 866 |
+
native = _cell_backend(table_key, st)
|
| 867 |
+
reader = getattr(native[0], 'get_json_paths', None) if native is not None else None
|
| 868 |
+
if not callable(reader):
|
| 869 |
+
return None
|
| 870 |
+
return (reader(native[1], [['_prune_pass']]) or [None])[0]
|
| 871 |
+
|
| 872 |
+
|
| 873 |
def prune(table_key, live_pids, st=None, *, pass_id=None, validate_source=None):
|
| 874 |
"""Drop shared cells for rows that no longer exist. Returns how many rows were dropped.
|
| 875 |
|
|
|
|
| 887 |
keep = {_pid(p) for p in live_pids}
|
| 888 |
if not keep:
|
| 889 |
return 0
|
| 890 |
+
if pass_id is not None and _stored_prune_pass(table_key, st) == pass_id:
|
| 891 |
+
return 0
|
| 892 |
dropped = []
|
| 893 |
completed = False
|
| 894 |
|
platform/modules/competitor_watch.py
CHANGED
|
@@ -46,6 +46,8 @@ READ_FAILED = ("The Amazon mirror could not be read ({cause}), so no search term
|
|
| 46 |
"and the next Amazon sync will retry.")
|
| 47 |
BA_DATES_MISSING = "Brand Analytics report dates are missing"
|
| 48 |
BA_WEEKLY_MISSING = "Brand Analytics has no dated weekly reports"
|
|
|
|
|
|
|
| 49 |
LIMIT = "Alerts and checks use separate store rows and server pages, with no row cap."
|
| 50 |
|
| 51 |
KEY_SETTINGS = "competitor_watch_settings"
|
|
@@ -442,7 +444,13 @@ def _columns(cur, table):
|
|
| 442 |
|
| 443 |
def _ba_columns(cur):
|
| 444 |
"""Check weekly readiness with metadata and scalar existence checks, without loading source rows."""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 445 |
cols = _columns(cur, BA_TABLE) if cur is not None else set()
|
|
|
|
| 446 |
if cols and not {"_period_start", "_period_end"} <= cols:
|
| 447 |
try:
|
| 448 |
populated = cur.execute(f"SELECT EXISTS(SELECT 1 FROM {BA_TABLE})").fetchone()[0]
|
|
@@ -459,7 +467,7 @@ def _ba_columns(cur):
|
|
| 459 |
raise _ReadFailed(type(exc).__name__) from exc
|
| 460 |
if populated and not weekly:
|
| 461 |
raise SourceNotReady(BA_WEEKLY_MISSING)
|
| 462 |
-
return cols
|
| 463 |
|
| 464 |
|
| 465 |
def _sqp(cur, term=None):
|
|
@@ -781,7 +789,9 @@ def term_page(tenant_key, *, st=None, search="", offset=0, limit=50, selected=No
|
|
| 781 |
source_error = None
|
| 782 |
try:
|
| 783 |
with _reader(tenant_key, st) as cur:
|
| 784 |
-
|
|
|
|
|
|
|
| 785 |
except SourceNotReady as exc:
|
| 786 |
source_error = exc
|
| 787 |
cause = READ_FAILED.format(cause=exc)
|
|
@@ -1034,6 +1044,12 @@ def on_quotes(tenant, changed):
|
|
| 1034 |
compare(tenant, ("daily_quotes",), st=st, quote_asins=changed)
|
| 1035 |
|
| 1036 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1037 |
def compare(tenant_key, synced_tables=("brand_analytics",), *, st=None, now=None, quote_asins=None):
|
| 1038 |
"""Compare the newest held week with the week before and raise owned alerts. Returns a summary."""
|
| 1039 |
st = _rt(tenant_key, st)
|
|
@@ -1065,6 +1081,9 @@ def compare(tenant_key, synced_tables=("brand_analytics",), *, st=None, now=None
|
|
| 1065 |
if daily and not retry:
|
| 1066 |
return {"state": "already_compared", "period": newest}
|
| 1067 |
if not daily and weeks.get("last_compared") == newest:
|
|
|
|
|
|
|
|
|
|
| 1068 |
return {"state": "already_compared", "period": newest}
|
| 1069 |
previous = periods[1] if len(periods) > 1 else None
|
| 1070 |
conf = settings(tenant_key, st=st)
|
|
@@ -1160,8 +1179,7 @@ def compare(tenant_key, synced_tables=("brand_analytics",), *, st=None, now=None
|
|
| 1160 |
"periodStart": newest, "previousPeriodStart": previous,
|
| 1161 |
"comparedAt": _stamp(now), "alertsRaised": len(created),
|
| 1162 |
"checks": len(checks)}
|
| 1163 |
-
|
| 1164 |
-
return doc
|
| 1165 |
st.update(KEY_WEEKS, _weeks)
|
| 1166 |
return {"state": "compared" if ready else "waiting", "period": newest, "previous": previous,
|
| 1167 |
"created": created, "checks": checks}
|
|
|
|
| 46 |
"and the next Amazon sync will retry.")
|
| 47 |
BA_DATES_MISSING = "Brand Analytics report dates are missing"
|
| 48 |
BA_WEEKLY_MISSING = "Brand Analytics has no dated weekly reports"
|
| 49 |
+
#: Stored sync failures that a live probe finding a dated weekly report proves superseded.
|
| 50 |
+
SUPERSEDED_BY_WEEKLY = frozenset(READ_FAILED.format(cause=cause) for cause in (BA_DATES_MISSING, BA_WEEKLY_MISSING))
|
| 51 |
LIMIT = "Alerts and checks use separate store rows and server pages, with no row cap."
|
| 52 |
|
| 53 |
KEY_SETTINGS = "competitor_watch_settings"
|
|
|
|
| 444 |
|
| 445 |
def _ba_columns(cur):
|
| 446 |
"""Check weekly readiness with metadata and scalar existence checks, without loading source rows."""
|
| 447 |
+
return _ba_source(cur)[0]
|
| 448 |
+
|
| 449 |
+
|
| 450 |
+
def _ba_source(cur):
|
| 451 |
+
"""The Brand Analytics columns and whether a dated weekly report exists. Unready sources raise."""
|
| 452 |
cols = _columns(cur, BA_TABLE) if cur is not None else set()
|
| 453 |
+
weekly = False
|
| 454 |
if cols and not {"_period_start", "_period_end"} <= cols:
|
| 455 |
try:
|
| 456 |
populated = cur.execute(f"SELECT EXISTS(SELECT 1 FROM {BA_TABLE})").fetchone()[0]
|
|
|
|
| 467 |
raise _ReadFailed(type(exc).__name__) from exc
|
| 468 |
if populated and not weekly:
|
| 469 |
raise SourceNotReady(BA_WEEKLY_MISSING)
|
| 470 |
+
return cols, bool(weekly)
|
| 471 |
|
| 472 |
|
| 473 |
def _sqp(cur, term=None):
|
|
|
|
| 789 |
source_error = None
|
| 790 |
try:
|
| 791 |
with _reader(tenant_key, st) as cur:
|
| 792 |
+
weekly = _ba_source(cur)[1]
|
| 793 |
+
if weekly:
|
| 794 |
+
problems = [problem for problem in problems if problem not in SUPERSEDED_BY_WEEKLY]
|
| 795 |
except SourceNotReady as exc:
|
| 796 |
source_error = exc
|
| 797 |
cause = READ_FAILED.format(cause=exc)
|
|
|
|
| 1044 |
compare(tenant, ("daily_quotes",), st=st, quote_asins=changed)
|
| 1045 |
|
| 1046 |
|
| 1047 |
+
def _clear_error(doc):
|
| 1048 |
+
doc = doc if isinstance(doc, dict) else {}
|
| 1049 |
+
doc.pop("last_error", None)
|
| 1050 |
+
return doc
|
| 1051 |
+
|
| 1052 |
+
|
| 1053 |
def compare(tenant_key, synced_tables=("brand_analytics",), *, st=None, now=None, quote_asins=None):
|
| 1054 |
"""Compare the newest held week with the week before and raise owned alerts. Returns a summary."""
|
| 1055 |
st = _rt(tenant_key, st)
|
|
|
|
| 1081 |
if daily and not retry:
|
| 1082 |
return {"state": "already_compared", "period": newest}
|
| 1083 |
if not daily and weeks.get("last_compared") == newest:
|
| 1084 |
+
# Every read this pass succeeded on a compared week, so a stored sync failure is superseded.
|
| 1085 |
+
if "last_error" in weeks:
|
| 1086 |
+
st.update(KEY_WEEKS, _clear_error)
|
| 1087 |
return {"state": "already_compared", "period": newest}
|
| 1088 |
previous = periods[1] if len(periods) > 1 else None
|
| 1089 |
conf = settings(tenant_key, st=st)
|
|
|
|
| 1179 |
"periodStart": newest, "previousPeriodStart": previous,
|
| 1180 |
"comparedAt": _stamp(now), "alertsRaised": len(created),
|
| 1181 |
"checks": len(checks)}
|
| 1182 |
+
return _clear_error(doc)
|
|
|
|
| 1183 |
st.update(KEY_WEEKS, _weeks)
|
| 1184 |
return {"state": "compared" if ready else "waiting", "period": newest, "previous": previous,
|
| 1185 |
"created": created, "checks": checks}
|
platform/modules/product_data.py
CHANGED
|
@@ -749,7 +749,8 @@ def source_batches(cur, *, st, team_id=None, ids=None, needed=None, raw_fields=(
|
|
| 749 |
with bounded(cur, 'product', [code for _, code in codes], st):
|
| 750 |
rows = ([{'pid': pid, 'code': code} for pid, code in codes]
|
| 751 |
if needed is not None and not needed else
|
| 752 |
-
pool(team_id=team_id, raw_fields=raw_fields, needed=needed
|
|
|
|
| 753 |
yield rows
|
| 754 |
rows.clear()
|
| 755 |
codes.clear()
|
|
@@ -854,7 +855,7 @@ def _fba_by_code(cat):
|
|
| 854 |
return out, report
|
| 855 |
|
| 856 |
|
| 857 |
-
def pool(team_id=None, t=None, raw_fields=(), *, needed=None):
|
| 858 |
"""One row per ACTIVE product — the PRODUCT analogue of `customer_data.pool`.
|
| 859 |
|
| 860 |
CATALOGUE-FIRST, REVENUE LEFT-JOINED (R12 — see decision 4 in the module header). The row set
|
|
@@ -865,6 +866,9 @@ def pool(team_id=None, t=None, raw_fields=(), *, needed=None):
|
|
| 865 |
passes it into `_sku_rev`), which is why `core.perm_scope.derive_pool_scope` must keep
|
| 866 |
driving it rather than a post-filter deciding the BU. It does NOT shape the row set: a
|
| 867 |
catalogue has no team.
|
|
|
|
|
|
|
|
|
|
| 868 |
"""
|
| 869 |
from harness import canonical_source, datastore
|
| 870 |
source = canonical_source.active()
|
|
@@ -1191,7 +1195,8 @@ def pool(team_id=None, t=None, raw_fields=(), *, needed=None):
|
|
| 1191 |
# nothing sold".
|
| 1192 |
rows.sort(key=lambda r: -(r["rev_ytd"] or 0.0))
|
| 1193 |
_assert_no_pid_collision(rows)
|
| 1194 |
-
|
|
|
|
| 1195 |
return rows
|
| 1196 |
|
| 1197 |
|
|
|
|
| 749 |
with bounded(cur, 'product', [code for _, code in codes], st):
|
| 750 |
rows = ([{'pid': pid, 'code': code} for pid, code in codes]
|
| 751 |
if needed is not None and not needed else
|
| 752 |
+
pool(team_id=team_id, raw_fields=raw_fields, needed=needed,
|
| 753 |
+
prune_overlay=ids is None))
|
| 754 |
yield rows
|
| 755 |
rows.clear()
|
| 756 |
codes.clear()
|
|
|
|
| 855 |
return out, report
|
| 856 |
|
| 857 |
|
| 858 |
+
def pool(team_id=None, t=None, raw_fields=(), *, needed=None, prune_overlay=True):
|
| 859 |
"""One row per ACTIVE product — the PRODUCT analogue of `customer_data.pool`.
|
| 860 |
|
| 861 |
CATALOGUE-FIRST, REVENUE LEFT-JOINED (R12 — see decision 4 in the module header). The row set
|
|
|
|
| 866 |
passes it into `_sku_rev`), which is why `core.perm_scope.derive_pool_scope` must keep
|
| 867 |
driving it rather than a post-filter deciding the BU. It does NOT shape the row set: a
|
| 868 |
catalogue has no team.
|
| 869 |
+
|
| 870 |
+
`prune_overlay=False` is a page read: the whole-catalogue overlay prune is a store write that
|
| 871 |
+
belongs to whole-catalogue reads, never to a request that hydrates a few selected rows.
|
| 872 |
"""
|
| 873 |
from harness import canonical_source, datastore
|
| 874 |
source = canonical_source.active()
|
|
|
|
| 1195 |
# nothing sold".
|
| 1196 |
rows.sort(key=lambda r: -(r["rev_ytd"] or 0.0))
|
| 1197 |
_assert_no_pid_collision(rows)
|
| 1198 |
+
if prune_overlay:
|
| 1199 |
+
_prune_product_overlay(rows, source, prune_generation)
|
| 1200 |
return rows
|
| 1201 |
|
| 1202 |
|
web/public/AIOS_DEPLOY_RECEIPT
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
|
|
|
|
| 1 |
+
ee4618c92d8cbec218349feb4f5f9fbc6cb19f8ba0f0e63314aaa24270742c00
|