Deploy AIOS web (React glide grid + FastAPI slice)
Browse files- RELEASES.json +1 -1
- VERSION +1 -1
- api/amazon_relational.py +1 -0
- api/routes_competitor_watch.py +5 -13
- platform/core/received_changes.py +194 -50
- platform/modules/competitor_watch.py +166 -107
- web/public/AIOS_DEPLOY_RECEIPT +1 -1
RELEASES.json
CHANGED
|
@@ -1,4 +1,4 @@
|
|
| 1 |
{
|
| 2 |
-
"current": "
|
| 3 |
"releases": []
|
| 4 |
}
|
|
|
|
| 1 |
{
|
| 2 |
+
"current": "2f4d90c03d",
|
| 3 |
"releases": []
|
| 4 |
}
|
VERSION
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
|
|
|
|
| 1 |
+
2f4d90c03d
|
api/amazon_relational.py
CHANGED
|
@@ -994,6 +994,7 @@ def publication_sources(cur, keys):
|
|
| 994 |
if schema is None:
|
| 995 |
continue
|
| 996 |
spec = grid_source(cur, key, allow_lookups=False, status=None, schema=schema)
|
|
|
|
| 997 |
if amazon_schema.CONNECTION_ID in dict(schema[0]):
|
| 998 |
spec['cols'][amazon_schema.CONNECTION_ID] = ('"_connection_id"', _text)
|
| 999 |
result[key] = spec
|
|
|
|
| 994 |
if schema is None:
|
| 995 |
continue
|
| 996 |
spec = grid_source(cur, key, allow_lookups=False, status=None, schema=schema)
|
| 997 |
+
spec['observation_tables'] = [SOURCE[key]]
|
| 998 |
if amazon_schema.CONNECTION_ID in dict(schema[0]):
|
| 999 |
spec['cols'][amazon_schema.CONNECTION_ID] = ('"_connection_id"', _text)
|
| 1000 |
result[key] = spec
|
api/routes_competitor_watch.py
CHANGED
|
@@ -217,21 +217,13 @@ def _not_ready_message(cw, periods):
|
|
| 217 |
return cw.NOT_READY
|
| 218 |
|
| 219 |
|
| 220 |
-
def _term_rows(cw, tenant, st, page):
|
| 221 |
"""Read dated-week counts for this server page without loading its ranking history."""
|
| 222 |
terms = page["terms"]
|
| 223 |
-
counts
|
| 224 |
-
if terms:
|
| 225 |
try:
|
| 226 |
with cw._reader(tenant, st) as cur:
|
| 227 |
-
|
| 228 |
-
if {"search_term", "_period_start", "_period_end"} <= cols:
|
| 229 |
-
key = cw._term_sql("search_term")
|
| 230 |
-
rows = cur.execute(
|
| 231 |
-
f'SELECT {key}, COUNT(DISTINCT CAST("_period_start" AS DATE)) '
|
| 232 |
-
f'FROM {cw.BA_TABLE} WHERE {key} IN (' + ','.join('?' for _ in terms) + ') '
|
| 233 |
-
f'AND ({cw._weekly_period_sql()}) GROUP BY {key}', terms).fetchall()
|
| 234 |
-
counts = dict(rows)
|
| 235 |
except Exception as exc: # noqa: BLE001
|
| 236 |
cause = str(exc) if isinstance(exc, cw._ReadFailed) else type(exc).__name__
|
| 237 |
raise cw.SourceNotReady(cause, page=page) from exc
|
|
@@ -315,6 +307,6 @@ def app_screen(app: str = Query(..., min_length=1), term: str | None = Query(Non
|
|
| 315 |
page = _call(cw.term_page, tenant, st=session.runtime, search=q, offset=offset,
|
| 316 |
limit=limit, selected=term, tracked=tracked, termOrder=termOrder)
|
| 317 |
terms, picked = page["terms"], page["term"]
|
| 318 |
-
|
| 319 |
-
|
| 320 |
return screen_payload(cw, detail, terms, picked, _may_edit(session, record), page=page)
|
|
|
|
| 217 |
return cw.NOT_READY
|
| 218 |
|
| 219 |
|
| 220 |
+
def _term_rows(cw, tenant, st, page, counts=None):
|
| 221 |
"""Read dated-week counts for this server page without loading its ranking history."""
|
| 222 |
terms = page["terms"]
|
| 223 |
+
if counts is None:
|
|
|
|
| 224 |
try:
|
| 225 |
with cw._reader(tenant, st) as cur:
|
| 226 |
+
counts, _history = cw._page_history(cur, terms, None)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 227 |
except Exception as exc: # noqa: BLE001
|
| 228 |
cause = str(exc) if isinstance(exc, cw._ReadFailed) else type(exc).__name__
|
| 229 |
raise cw.SourceNotReady(cause, page=page) from exc
|
|
|
|
| 307 |
page = _call(cw.term_page, tenant, st=session.runtime, search=q, offset=offset,
|
| 308 |
limit=limit, selected=term, tracked=tracked, termOrder=termOrder)
|
| 309 |
terms, picked = page["terms"], page["term"]
|
| 310 |
+
counts, detail = _call(cw.term_page_history, tenant, page, st=session.runtime)
|
| 311 |
+
page = {**page, "termRows": _term_rows(cw, tenant, session.runtime, page, counts)}
|
| 312 |
return screen_payload(cw, detail, terms, picked, _may_edit(session, record), page=page)
|
platform/core/received_changes.py
CHANGED
|
@@ -24,6 +24,7 @@ PROJECTION_VERSION = 1
|
|
| 24 |
IMAGE_DIGEST = "md5_number(image)"
|
| 25 |
_LISTENERS = []
|
| 26 |
_PROJECTIONS = {}
|
|
|
|
| 27 |
_LOCK = threading.RLock()
|
| 28 |
_LOG = logging.getLogger(__name__)
|
| 29 |
|
|
@@ -313,6 +314,8 @@ def _write_blob(rt, path, data):
|
|
| 313 |
"""Reuse immutable content and verify persistence before publishing its reference."""
|
| 314 |
from core import blob_envelope as be
|
| 315 |
digest, size = _digest(data), len(data)
|
|
|
|
|
|
|
| 316 |
raw = rt.download_bytes(path)
|
| 317 |
if raw is None:
|
| 318 |
stored = be.seal(rt.key, path, data)
|
|
@@ -324,18 +327,45 @@ def _write_blob(rt, path, data):
|
|
| 324 |
return {"sha256": digest, "bytes": size}
|
| 325 |
|
| 326 |
|
| 327 |
-
def
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 328 |
from core import blob_envelope as be
|
| 329 |
parts, content, size = [], hashlib.sha256(), 0
|
|
|
|
| 330 |
with path.open("rb") as handle:
|
| 331 |
while chunk := handle.read(BASELINE_CHUNK_BYTES - len(be.HEADER) - 28):
|
| 332 |
-
|
|
|
|
|
|
|
| 333 |
content.update(chunk)
|
| 334 |
size += len(chunk)
|
| 335 |
-
manifest = {**binding, "snapshot": snapshot, "at": stamp, "parts": parts,
|
| 336 |
"sha256": content.hexdigest(), "bytes": size}
|
| 337 |
data = _json_bytes(manifest)
|
| 338 |
-
return _write_blob(rt, _blob_path(identity, "manifests", _digest(data)), data)
|
| 339 |
|
| 340 |
|
| 341 |
def _cached_baseline(cache, identity, reference):
|
|
@@ -348,6 +378,7 @@ def _cache_baseline(cache, identity, reference, path):
|
|
| 348 |
for stale in cache.glob(identity + "-*.parquet"):
|
| 349 |
stale.unlink(missing_ok=True)
|
| 350 |
os.replace(path, _cached_baseline(cache, identity, reference))
|
|
|
|
| 351 |
|
| 352 |
|
| 353 |
def _restore_baseline(rt, identity, binding, reference, path, con, cache):
|
|
@@ -355,11 +386,21 @@ def _restore_baseline(rt, identity, binding, reference, path, con, cache):
|
|
| 355 |
data = _read_blob(rt, _blob_path(identity, "manifests", reference["sha256"]),
|
| 356 |
reference["sha256"], reference["bytes"])
|
| 357 |
manifest = json.loads(data)
|
| 358 |
-
if (any(manifest.get(key) != value for key, value in binding.items())
|
|
|
|
| 359 |
or not isinstance(manifest.get("snapshot"), str) or not manifest["snapshot"]
|
| 360 |
or not isinstance(manifest.get("parts"), list) or not manifest["parts"]
|
| 361 |
or not manifest.get("at")):
|
| 362 |
raise BaselineUnavailable("The observation manifest has a different identity or shape.")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 363 |
_at(manifest["at"])
|
| 364 |
cached = _cached_baseline(cache, identity, reference)
|
| 365 |
if cached.exists():
|
|
@@ -374,7 +415,8 @@ def _restore_baseline(rt, identity, binding, reference, path, con, cache):
|
|
| 374 |
for part in manifest["parts"]))
|
| 375 |
if content != manifest["sha256"] or size != manifest["bytes"]:
|
| 376 |
raise BaselineUnavailable("The observation snapshot failed content verification.")
|
| 377 |
-
|
|
|
|
| 378 |
return manifest
|
| 379 |
except BaselineUnavailable:
|
| 380 |
raise
|
|
@@ -428,8 +470,8 @@ def _project(con, query, path, database):
|
|
| 428 |
raw.unlink()
|
| 429 |
|
| 430 |
|
| 431 |
-
def _observation_state(rt, identity):
|
| 432 |
-
document = _document(rt, rt.get(STORE_KEY, fresh=True))
|
| 433 |
states = document.get("observations", {})
|
| 434 |
if not isinstance(states, dict):
|
| 435 |
raise BaselineUnavailable("The observation index is invalid.")
|
|
@@ -463,6 +505,7 @@ def _move_baseline(rt, identity, expected, updated, observed_at=None):
|
|
| 463 |
result = rt.update(STORE_KEY, move, flush="sync")
|
| 464 |
if result is None or result is False:
|
| 465 |
raise BaselineUnavailable("The observation reference was not saved.")
|
|
|
|
| 466 |
|
| 467 |
|
| 468 |
@contextmanager
|
|
@@ -485,14 +528,16 @@ def _compare_baseline(rt, source, database, spec, con, baseline, current, snapsh
|
|
| 485 |
changed = (f"SELECT coalesce(b.row_id, a.row_id) FROM {digests.format('$1')} b FULL OUTER JOIN "
|
| 486 |
f"{digests.format('$2')} a USING (row_id) WHERE b.image IS DISTINCT FROM a.image")
|
| 487 |
files = [str(baseline), str(current)]
|
| 488 |
-
|
|
|
|
| 489 |
# A diff wider than the retained window cannot be replayed, so it is recorded as one bulk change.
|
| 490 |
-
if
|
| 491 |
-
count = con.execute(f"SELECT count(*) FROM ({changed})", files).fetchone()[0]
|
| 492 |
print(f"[received-changes] {database}: {count} changed rows exceed the journal's "
|
| 493 |
f"{MAX_EVENTS}-event window; recorded as a bulk change, rows not journaled", flush=True)
|
| 494 |
return
|
| 495 |
-
|
|
|
|
|
|
|
| 496 |
try:
|
| 497 |
with _query_watchdog(cursor, deadline):
|
| 498 |
picked = "(SELECT * FROM read_parquet({}) WHERE row_id IN (SELECT unnest($3::VARCHAR[])))"
|
|
@@ -527,7 +572,40 @@ def _observation_identity(rt, source, database):
|
|
| 527 |
return hashlib.sha256(json.dumps([_tenant(rt), source, database]).encode()).hexdigest()
|
| 528 |
|
| 529 |
|
| 530 |
-
def
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 531 |
"""Compare verified durable observations in batches, recovering interrupted comparisons first."""
|
| 532 |
if not spec.get("cols"):
|
| 533 |
return
|
|
@@ -540,7 +618,7 @@ def _snapshot(rt, source, database, spec, con, root, manifest, deadline=None):
|
|
| 540 |
if not stamp:
|
| 541 |
raise BaselineUnavailable("Connector observations require a completed publication time.")
|
| 542 |
stamp = _at(stamp)
|
| 543 |
-
state, document = _observation_state(rt, identity)
|
| 544 |
if state is None and (any((root / (identity + suffix)).exists() for suffix in (".parquet", ".json"))
|
| 545 |
or any(event["source"] == source and event["database"] == database
|
| 546 |
for event in document["events"])):
|
|
@@ -549,46 +627,105 @@ def _snapshot(rt, source, database, spec, con, root, manifest, deadline=None):
|
|
| 549 |
with tempfile.TemporaryDirectory(dir=root, prefix="projection-") as directory:
|
| 550 |
baseline, current = Path(directory) / "baseline.parquet", Path(directory) / "current.parquet"
|
| 551 |
|
| 552 |
-
|
| 553 |
-
|
| 554 |
-
|
| 555 |
-
|
| 556 |
-
|
| 557 |
-
|
| 558 |
-
|
| 559 |
-
|
| 560 |
-
|
| 561 |
-
|
| 562 |
-
|
| 563 |
-
|
| 564 |
-
|
| 565 |
-
|
| 566 |
-
if state and state["pending"]:
|
| 567 |
-
state = complete(state)
|
| 568 |
-
if state:
|
| 569 |
-
prior = _restore_baseline(rt, identity, binding, state["current"], baseline, con, cache)
|
| 570 |
prior_at = dt.datetime.fromisoformat(prior["at"].replace("Z", "+00:00"))
|
| 571 |
incoming_at = dt.datetime.fromisoformat(stamp.replace("Z", "+00:00"))
|
| 572 |
-
if
|
|
|
|
|
|
|
| 573 |
return
|
| 574 |
-
if incoming_at == prior_at:
|
| 575 |
raise BaselineUnavailable("Distinct connector snapshots have ambiguous publication order.")
|
| 576 |
-
|
| 577 |
-
|
| 578 |
-
|
| 579 |
-
|
| 580 |
-
|
| 581 |
-
|
| 582 |
-
|
| 583 |
-
|
| 584 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 585 |
_move_baseline(rt, identity, state, pending)
|
| 586 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 587 |
|
| 588 |
|
| 589 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 590 |
"""Least recently observed databases first, then the smallest, so a deadline defers in rotation."""
|
| 591 |
-
|
|
|
|
| 592 |
|
| 593 |
def priority(item):
|
| 594 |
database, spec = item
|
|
@@ -598,13 +735,17 @@ def _publication_order(rt, connector, specs, paths):
|
|
| 598 |
return sorted(specs.items(), key=priority)
|
| 599 |
|
| 600 |
|
| 601 |
-
def _project_database(rt, connector, database, spec, con, root, manifest, deadline):
|
| 602 |
"""Return None when the database committed, else the named reason it was deferred."""
|
| 603 |
import duckdb
|
| 604 |
from core import request_timing
|
|
|
|
| 605 |
try:
|
| 606 |
with _query_watchdog(con, deadline):
|
| 607 |
-
|
|
|
|
|
|
|
|
|
|
| 608 |
except request_timing.RequestDeadlineExceeded as exc:
|
| 609 |
# The watchdog raises on exit after a clean body too; no cause means the database committed.
|
| 610 |
return None if exc.__cause__ is None else "journal deadline passed"
|
|
@@ -613,6 +754,8 @@ def _project_database(rt, connector, database, spec, con, root, manifest, deadli
|
|
| 613 |
except duckdb.Error as exc:
|
| 614 |
# One database's failed query defers it alone; its stored baseline stays for the next pass.
|
| 615 |
return type(exc).__name__
|
|
|
|
|
|
|
| 616 |
return None
|
| 617 |
|
| 618 |
|
|
@@ -643,13 +786,14 @@ def mirror_hook(connector, tenant_key, manifest, _state_rows, paths):
|
|
| 643 |
resolution_seconds = time.monotonic() - started
|
| 644 |
snapshots_started = time.monotonic()
|
| 645 |
completed = deferred = skipped = 0
|
| 646 |
-
|
|
|
|
| 647 |
# A skipped database keeps its stored baseline and leads the next publication.
|
| 648 |
if time.monotonic() >= deadline[0]:
|
| 649 |
skipped += 1
|
| 650 |
print(f"[received-changes] {database}: skipped, journal deadline passed", flush=True)
|
| 651 |
continue
|
| 652 |
-
reason = _project_database(rt, connector, str(database), spec, con, root, manifest, deadline)
|
| 653 |
if reason is None:
|
| 654 |
completed += 1
|
| 655 |
continue
|
|
|
|
| 24 |
IMAGE_DIGEST = "md5_number(image)"
|
| 25 |
_LISTENERS = []
|
| 26 |
_PROJECTIONS = {}
|
| 27 |
+
_VERIFIED_BASELINES = {}
|
| 28 |
_LOCK = threading.RLock()
|
| 29 |
_LOG = logging.getLogger(__name__)
|
| 30 |
|
|
|
|
| 314 |
"""Reuse immutable content and verify persistence before publishing its reference."""
|
| 315 |
from core import blob_envelope as be
|
| 316 |
digest, size = _digest(data), len(data)
|
| 317 |
+
if _insert_blob(rt, path, data):
|
| 318 |
+
return {"sha256": digest, "bytes": size}
|
| 319 |
raw = rt.download_bytes(path)
|
| 320 |
if raw is None:
|
| 321 |
stored = be.seal(rt.key, path, data)
|
|
|
|
| 327 |
return {"sha256": digest, "bytes": size}
|
| 328 |
|
| 329 |
|
| 330 |
+
def _insert_blob(rt, path, data):
|
| 331 |
+
"""Verify a new Postgres blob in its insert transaction without transferring it back."""
|
| 332 |
+
from core import blob_envelope as be, store_pg
|
| 333 |
+
backend = rt._document_store() if hasattr(rt, "_document_store") else rt
|
| 334 |
+
if not isinstance(backend, store_pg.PgStore):
|
| 335 |
+
return False
|
| 336 |
+
if backend.tenant_slug != _tenant(rt):
|
| 337 |
+
raise BaselineUnavailable("The observation store belongs to another tenant.")
|
| 338 |
+
store_pg._effective_slug(backend.tenant_slug)
|
| 339 |
+
store_pg.check_write(backend.tenant_slug, "write received-change observation")
|
| 340 |
+
stored = be.seal(rt.key, path, data)
|
| 341 |
+
if len(stored) > BASELINE_CHUNK_BYTES:
|
| 342 |
+
raise BaselineUnavailable("The observation blob exceeds the store chunk size.")
|
| 343 |
+
name = rt.store_key(path) if hasattr(rt, "store_key") else path
|
| 344 |
+
with store_pg._pool().connection() as con:
|
| 345 |
+
row = con.execute(store_pg._q(
|
| 346 |
+
"INSERT INTO {tbl} (path, bytes) VALUES (%s, %s) ON CONFLICT (path) DO NOTHING "
|
| 347 |
+
"RETURNING octet_length(bytes), encode(sha256(bytes), 'hex')",
|
| 348 |
+
"store_blobs", backend.tenant_slug), [name, stored]).fetchone()
|
| 349 |
+
if row is not None and tuple(row) != (len(stored), _digest(stored)):
|
| 350 |
+
raise BaselineUnavailable("The observation upload failed content verification.")
|
| 351 |
+
return row is not None
|
| 352 |
+
|
| 353 |
+
|
| 354 |
+
def _persist_baseline(rt, identity, binding, path, snapshot, stamp, prior=None, **metadata):
|
| 355 |
from core import blob_envelope as be
|
| 356 |
parts, content, size = [], hashlib.sha256(), 0
|
| 357 |
+
verified = {(part["sha256"], part["bytes"]) for part in (prior or {}).get("parts", [])}
|
| 358 |
with path.open("rb") as handle:
|
| 359 |
while chunk := handle.read(BASELINE_CHUNK_BYTES - len(be.HEADER) - 28):
|
| 360 |
+
digest = _digest(chunk)
|
| 361 |
+
parts.append({"sha256": digest, "bytes": len(chunk)} if (digest, len(chunk)) in verified else
|
| 362 |
+
_write_blob(rt, _blob_path(identity, "chunks", digest), chunk))
|
| 363 |
content.update(chunk)
|
| 364 |
size += len(chunk)
|
| 365 |
+
manifest = {**binding, **metadata, "snapshot": snapshot, "at": stamp, "parts": parts,
|
| 366 |
"sha256": content.hexdigest(), "bytes": size}
|
| 367 |
data = _json_bytes(manifest)
|
| 368 |
+
return _write_blob(rt, _blob_path(identity, "manifests", _digest(data)), data), manifest
|
| 369 |
|
| 370 |
|
| 371 |
def _cached_baseline(cache, identity, reference):
|
|
|
|
| 378 |
for stale in cache.glob(identity + "-*.parquet"):
|
| 379 |
stale.unlink(missing_ok=True)
|
| 380 |
os.replace(path, _cached_baseline(cache, identity, reference))
|
| 381 |
+
_VERIFIED_BASELINES[(str(cache), identity)] = reference["sha256"]
|
| 382 |
|
| 383 |
|
| 384 |
def _restore_baseline(rt, identity, binding, reference, path, con, cache):
|
|
|
|
| 386 |
data = _read_blob(rt, _blob_path(identity, "manifests", reference["sha256"]),
|
| 387 |
reference["sha256"], reference["bytes"])
|
| 388 |
manifest = json.loads(data)
|
| 389 |
+
if (any(manifest.get(key) != value for key, value in binding.items() if key != "projection")
|
| 390 |
+
or not isinstance(manifest.get("projection"), str)
|
| 391 |
or not isinstance(manifest.get("snapshot"), str) or not manifest["snapshot"]
|
| 392 |
or not isinstance(manifest.get("parts"), list) or not manifest["parts"]
|
| 393 |
or not manifest.get("at")):
|
| 394 |
raise BaselineUnavailable("The observation manifest has a different identity or shape.")
|
| 395 |
+
_blob_path(identity, "projections", manifest["projection"])
|
| 396 |
+
for part in manifest["parts"]:
|
| 397 |
+
if (set(part) != {"sha256", "bytes"} or type(part["bytes"]) is not int
|
| 398 |
+
or not 0 < part["bytes"] <= BASELINE_CHUNK_BYTES):
|
| 399 |
+
raise BaselineUnavailable("The observation chunk reference is invalid.")
|
| 400 |
+
_blob_path(identity, "chunks", part["sha256"])
|
| 401 |
+
if (type(manifest.get("bytes")) is not int or manifest["bytes"] <= 0
|
| 402 |
+
or sum(part["bytes"] for part in manifest["parts"]) != manifest["bytes"]):
|
| 403 |
+
raise BaselineUnavailable("The observation content length is invalid.")
|
| 404 |
_at(manifest["at"])
|
| 405 |
cached = _cached_baseline(cache, identity, reference)
|
| 406 |
if cached.exists():
|
|
|
|
| 415 |
for part in manifest["parts"]))
|
| 416 |
if content != manifest["sha256"] or size != manifest["bytes"]:
|
| 417 |
raise BaselineUnavailable("The observation snapshot failed content verification.")
|
| 418 |
+
if _VERIFIED_BASELINES.get((str(cache), identity)) != reference["sha256"]:
|
| 419 |
+
_validate_projection(con, path)
|
| 420 |
return manifest
|
| 421 |
except BaselineUnavailable:
|
| 422 |
raise
|
|
|
|
| 470 |
raw.unlink()
|
| 471 |
|
| 472 |
|
| 473 |
+
def _observation_state(rt, identity, document=None):
|
| 474 |
+
document = _document(rt, rt.get(STORE_KEY, fresh=True)) if document is None else document
|
| 475 |
states = document.get("observations", {})
|
| 476 |
if not isinstance(states, dict):
|
| 477 |
raise BaselineUnavailable("The observation index is invalid.")
|
|
|
|
| 505 |
result = rt.update(STORE_KEY, move, flush="sync")
|
| 506 |
if result is None or result is False:
|
| 507 |
raise BaselineUnavailable("The observation reference was not saved.")
|
| 508 |
+
return result
|
| 509 |
|
| 510 |
|
| 511 |
@contextmanager
|
|
|
|
| 528 |
changed = (f"SELECT coalesce(b.row_id, a.row_id) FROM {digests.format('$1')} b FULL OUTER JOIN "
|
| 529 |
f"{digests.format('$2')} a USING (row_id) WHERE b.image IS DISTINCT FROM a.image")
|
| 530 |
files = [str(baseline), str(current)]
|
| 531 |
+
rows = con.execute(f"SELECT count(*) OVER (), * FROM ({changed}) LIMIT {MAX_EVENTS + 1}", files).fetchall()
|
| 532 |
+
count, ids = (rows[0][0] if rows else 0), [row[1] for row in rows]
|
| 533 |
# A diff wider than the retained window cannot be replayed, so it is recorded as one bulk change.
|
| 534 |
+
if count > MAX_EVENTS:
|
|
|
|
| 535 |
print(f"[received-changes] {database}: {count} changed rows exceed the journal's "
|
| 536 |
f"{MAX_EVENTS}-event window; recorded as a bulk change, rows not journaled", flush=True)
|
| 537 |
return
|
| 538 |
+
if not ids:
|
| 539 |
+
return
|
| 540 |
+
cursor, done = con.cursor(), 0
|
| 541 |
try:
|
| 542 |
with _query_watchdog(cursor, deadline):
|
| 543 |
picked = "(SELECT * FROM read_parquet({}) WHERE row_id IN (SELECT unnest($3::VARCHAR[])))"
|
|
|
|
| 572 |
return hashlib.sha256(json.dumps([_tenant(rt), source, database]).encode()).hexdigest()
|
| 573 |
|
| 574 |
|
| 575 |
+
def _complete_baseline(rt, identity, state, current, baseline, con, cache,
|
| 576 |
+
source, database, spec, deadline, observed, expected, unchanged=False, reset=False):
|
| 577 |
+
if state["current"] and not unchanged:
|
| 578 |
+
_compare_baseline(rt, source, database, spec, con, baseline, current, observed, deadline)
|
| 579 |
+
committed = {"current": state["pending"], "pending": None}
|
| 580 |
+
document = _move_baseline(rt, identity, expected, committed, observed_at=_at())
|
| 581 |
+
_cache_baseline(cache, identity, state["pending"], current)
|
| 582 |
+
if reset:
|
| 583 |
+
print(f"[received-changes] {database}: projection binding changed; new baseline, no events", flush=True)
|
| 584 |
+
return committed, document
|
| 585 |
+
|
| 586 |
+
|
| 587 |
+
def _prior_observations(rt, identity, binding, state, baseline, current, con, cache):
|
| 588 |
+
"""Verify both references before recovering or replacing a changed projection."""
|
| 589 |
+
held = {}
|
| 590 |
+
for key, path in (("current", baseline), ("pending", current)):
|
| 591 |
+
if state and state[key]:
|
| 592 |
+
held[key] = _restore_baseline(rt, identity, binding, state[key], path, con, cache)
|
| 593 |
+
if len(held) == 2:
|
| 594 |
+
before, after = held["current"], held["pending"]
|
| 595 |
+
if dt.datetime.fromisoformat(after["at"].replace("Z", "+00:00")) <= dt.datetime.fromisoformat(
|
| 596 |
+
before["at"].replace("Z", "+00:00")):
|
| 597 |
+
raise BaselineUnavailable("The pending observation has invalid publication order.")
|
| 598 |
+
return held
|
| 599 |
+
|
| 600 |
+
|
| 601 |
+
def _refresh_observation(rt, identity, binding, state, prior, snapshot, stamp):
|
| 602 |
+
"""Advance an identical input's publication order while retaining its verified content."""
|
| 603 |
+
data = _json_bytes({**prior, **binding, "snapshot": snapshot, "at": stamp})
|
| 604 |
+
reference = _write_blob(rt, _blob_path(identity, "manifests", _digest(data)), data)
|
| 605 |
+
return _move_baseline(rt, identity, state, {"current": reference, "pending": None}, observed_at=_at())
|
| 606 |
+
|
| 607 |
+
|
| 608 |
+
def _snapshot(rt, source, database, spec, con, root, manifest, deadline=None, publication=None):
|
| 609 |
"""Compare verified durable observations in batches, recovering interrupted comparisons first."""
|
| 610 |
if not spec.get("cols"):
|
| 611 |
return
|
|
|
|
| 618 |
if not stamp:
|
| 619 |
raise BaselineUnavailable("Connector observations require a completed publication time.")
|
| 620 |
stamp = _at(stamp)
|
| 621 |
+
state, document = _observation_state(rt, identity, publication.document if publication else None)
|
| 622 |
if state is None and (any((root / (identity + suffix)).exists() for suffix in (".parquet", ".json"))
|
| 623 |
or any(event["source"] == source and event["database"] == database
|
| 624 |
for event in document["events"])):
|
|
|
|
| 627 |
with tempfile.TemporaryDirectory(dir=root, prefix="projection-") as directory:
|
| 628 |
baseline, current = Path(directory) / "baseline.parquet", Path(directory) / "current.parquet"
|
| 629 |
|
| 630 |
+
restoring = time.monotonic()
|
| 631 |
+
held = _prior_observations(rt, identity, binding, state, baseline, current, con, cache)
|
| 632 |
+
print(f"[received-changes] {database}: restore_verify={time.monotonic() - restoring:.3f}s", flush=True)
|
| 633 |
+
reset = any(prior["projection"] != binding["projection"] for prior in held.values())
|
| 634 |
+
prior = held.get("pending") or held.get("current")
|
| 635 |
+
if state and state["pending"] and not reset:
|
| 636 |
+
state, document = _complete_baseline(rt, identity, state, current, baseline, con,
|
| 637 |
+
cache, source, database, spec, deadline, prior["snapshot"],
|
| 638 |
+
expected=state,
|
| 639 |
+
reset=bool(prior.get("reset_from")) and not state["current"])
|
| 640 |
+
# The recovered pending file became the cache, and is the next comparison's baseline.
|
| 641 |
+
baseline.unlink(missing_ok=True)
|
| 642 |
+
baseline.hardlink_to(_cached_baseline(cache, identity, state["current"]))
|
| 643 |
+
if prior:
|
|
|
|
|
|
|
|
|
|
|
|
|
| 644 |
prior_at = dt.datetime.fromisoformat(prior["at"].replace("Z", "+00:00"))
|
| 645 |
incoming_at = dt.datetime.fromisoformat(stamp.replace("Z", "+00:00"))
|
| 646 |
+
if incoming_at < prior_at:
|
| 647 |
+
if publication:
|
| 648 |
+
publication.document = document
|
| 649 |
return
|
| 650 |
+
if incoming_at == prior_at and prior["snapshot"] != snapshot:
|
| 651 |
raise BaselineUnavailable("Distinct connector snapshots have ambiguous publication order.")
|
| 652 |
+
if prior and not reset and prior["snapshot"] == snapshot:
|
| 653 |
+
if publication:
|
| 654 |
+
publication.document = document
|
| 655 |
+
return
|
| 656 |
+
if prior and not reset and spec.get("input_digest") == prior.get("input") and spec.get("input_digest"):
|
| 657 |
+
document = _refresh_observation(rt, identity, binding, state, prior, snapshot, stamp)
|
| 658 |
+
_cache_baseline(cache, identity, document["observations"][identity]["current"], baseline)
|
| 659 |
+
else:
|
| 660 |
+
document = _new_observation(rt, identity, binding, state, spec, con, current, baseline,
|
| 661 |
+
cache, source, database, snapshot, stamp, deadline, reset, prior)
|
| 662 |
+
if publication:
|
| 663 |
+
publication.document = document
|
| 664 |
+
|
| 665 |
+
|
| 666 |
+
def _new_observation(rt, identity, binding, state, spec, con, current, baseline, cache,
|
| 667 |
+
source, database, snapshot, stamp, deadline, reset, prior):
|
| 668 |
+
expressions = ", ".join(f"{_quoted(key)} := {value[0]}" for key, value in spec["cols"].items())
|
| 669 |
+
where = " WHERE " + spec["where"] if spec.get("where") else ""
|
| 670 |
+
query = (f"SELECT CAST(({spec['id']}) AS VARCHAR) AS row_id, "
|
| 671 |
+
f"CAST(to_json(struct_pack({expressions})) AS VARCHAR) AS image "
|
| 672 |
+
f"FROM {spec.get('from_sql') or spec['table']}{where}")
|
| 673 |
+
current.unlink(missing_ok=True)
|
| 674 |
+
started = time.monotonic()
|
| 675 |
+
_project(con, query, current, database)
|
| 676 |
+
projected = time.monotonic()
|
| 677 |
+
reference, observed = _persist_baseline(rt, identity, binding, current, snapshot, stamp,
|
| 678 |
+
prior=prior, input=spec.get("input_digest"), reset_from=prior["projection"] if reset else None)
|
| 679 |
+
persisted = time.monotonic()
|
| 680 |
+
pending = {"current": state["current"] if state and not reset else None, "pending": reference}
|
| 681 |
+
unchanged = bool(prior and prior["sha256"] == observed["sha256"])
|
| 682 |
+
expected = state
|
| 683 |
+
# Only a comparison that can emit events needs a durable pending checkpoint.
|
| 684 |
+
if pending["current"] and not unchanged:
|
| 685 |
_move_baseline(rt, identity, state, pending)
|
| 686 |
+
expected = pending
|
| 687 |
+
_committed, document = _complete_baseline(rt, identity, pending, current, baseline,
|
| 688 |
+
con, cache, source, database, spec, deadline, snapshot, expected=expected,
|
| 689 |
+
unchanged=unchanged, reset=reset)
|
| 690 |
+
print(f"[received-changes] {database}: project={projected - started:.3f}s "
|
| 691 |
+
f"persist={persisted - projected:.3f}s compare_commit={time.monotonic() - persisted:.3f}s", flush=True)
|
| 692 |
+
return document
|
| 693 |
|
| 694 |
|
| 695 |
+
class _ObservationPublication:
|
| 696 |
+
"""Share one authoritative journal read and ordered input digests across a publication."""
|
| 697 |
+
|
| 698 |
+
def __init__(self, rt, paths):
|
| 699 |
+
started = time.monotonic()
|
| 700 |
+
self.document = _document(rt, rt.get(STORE_KEY, fresh=True))
|
| 701 |
+
self.paths, self.digests = paths, {}
|
| 702 |
+
print(f"[received-changes] journal_read={time.monotonic() - started:.3f}s", flush=True)
|
| 703 |
+
|
| 704 |
+
def input_digest(self, spec, deadline):
|
| 705 |
+
tables = spec.get("observation_tables", sorted(self.paths))
|
| 706 |
+
parts = []
|
| 707 |
+
for table in tables:
|
| 708 |
+
files = self.paths.get(table)
|
| 709 |
+
if not files:
|
| 710 |
+
return None
|
| 711 |
+
for path in files:
|
| 712 |
+
name = str(path)
|
| 713 |
+
if name not in self.digests:
|
| 714 |
+
content = hashlib.sha256()
|
| 715 |
+
with open(path, "rb") as handle:
|
| 716 |
+
while chunk := handle.read(BASELINE_CHUNK_BYTES):
|
| 717 |
+
if time.monotonic() >= deadline[0]:
|
| 718 |
+
raise BaselineUnavailable("The observation input deadline passed.")
|
| 719 |
+
content.update(chunk)
|
| 720 |
+
self.digests[name] = content.hexdigest()
|
| 721 |
+
parts.append([table, self.digests[name]])
|
| 722 |
+
return _digest(_json_bytes(parts))
|
| 723 |
+
|
| 724 |
+
|
| 725 |
+
def _publication_order(rt, connector, specs, paths, document=None):
|
| 726 |
"""Least recently observed databases first, then the smallest, so a deadline defers in rotation."""
|
| 727 |
+
document = _document(rt, rt.get(STORE_KEY, fresh=True)) if document is None else document
|
| 728 |
+
observed = document.get("observed") or {}
|
| 729 |
|
| 730 |
def priority(item):
|
| 731 |
database, spec = item
|
|
|
|
| 735 |
return sorted(specs.items(), key=priority)
|
| 736 |
|
| 737 |
|
| 738 |
+
def _project_database(rt, connector, database, spec, con, root, manifest, deadline, publication=None):
|
| 739 |
"""Return None when the database committed, else the named reason it was deferred."""
|
| 740 |
import duckdb
|
| 741 |
from core import request_timing
|
| 742 |
+
started = time.monotonic()
|
| 743 |
try:
|
| 744 |
with _query_watchdog(con, deadline):
|
| 745 |
+
if publication:
|
| 746 |
+
spec = {**spec, "input_digest": publication.input_digest(spec, deadline)}
|
| 747 |
+
print(f"[received-changes] {database}: input_hash={time.monotonic() - started:.3f}s", flush=True)
|
| 748 |
+
_snapshot(rt, connector, database, spec, con, root, manifest, deadline, publication)
|
| 749 |
except request_timing.RequestDeadlineExceeded as exc:
|
| 750 |
# The watchdog raises on exit after a clean body too; no cause means the database committed.
|
| 751 |
return None if exc.__cause__ is None else "journal deadline passed"
|
|
|
|
| 754 |
except duckdb.Error as exc:
|
| 755 |
# One database's failed query defers it alone; its stored baseline stays for the next pass.
|
| 756 |
return type(exc).__name__
|
| 757 |
+
finally:
|
| 758 |
+
print(f"[received-changes] {database}: total={time.monotonic() - started:.3f}s", flush=True)
|
| 759 |
return None
|
| 760 |
|
| 761 |
|
|
|
|
| 786 |
resolution_seconds = time.monotonic() - started
|
| 787 |
snapshots_started = time.monotonic()
|
| 788 |
completed = deferred = skipped = 0
|
| 789 |
+
publication = _ObservationPublication(rt, paths)
|
| 790 |
+
for database, spec in _publication_order(rt, connector, specs, paths, publication.document):
|
| 791 |
# A skipped database keeps its stored baseline and leads the next publication.
|
| 792 |
if time.monotonic() >= deadline[0]:
|
| 793 |
skipped += 1
|
| 794 |
print(f"[received-changes] {database}: skipped, journal deadline passed", flush=True)
|
| 795 |
continue
|
| 796 |
+
reason = _project_database(rt, connector, str(database), spec, con, root, manifest, deadline, publication)
|
| 797 |
if reason is None:
|
| 798 |
completed += 1
|
| 799 |
continue
|
platform/modules/competitor_watch.py
CHANGED
|
@@ -231,12 +231,11 @@ class _TermIndex(_DocumentIndex):
|
|
| 231 |
self._put_meta(con, meta)
|
| 232 |
|
| 233 |
def _insert(self, con, rows):
|
| 234 |
-
if not rows:
|
| 235 |
-
return
|
| 236 |
statement = self.term_sql("INSERT INTO {terms} (tenant, term, label, active, starred) "
|
| 237 |
-
"
|
| 238 |
-
|
| 239 |
-
|
|
|
|
| 240 |
|
| 241 |
def seed(self, found, legacy, *, ours=None, tracking_error=None):
|
| 242 |
with self._locked(prepare=True) as (con, meta):
|
|
@@ -265,13 +264,9 @@ class _TermIndex(_DocumentIndex):
|
|
| 265 |
self._insert(con, [(self.slug, t, display_term(t), True, False) for t in found if t])
|
| 266 |
# Only a checked ownership result may replace membership, including a valid empty set.
|
| 267 |
if ours is not None:
|
| 268 |
-
con.execute(self.term_sql("UPDATE {terms} SET ours = FALSE WHERE tenant = %s AND ours = TRUE"),
|
| 269 |
-
[self.slug])
|
| 270 |
self._insert(con, [(self.slug, t, display_term(t), True, False) for t in ours])
|
| 271 |
-
|
| 272 |
-
|
| 273 |
-
cur.executemany(self.term_sql("UPDATE {terms} SET ours = TRUE WHERE tenant = %s AND term = %s"),
|
| 274 |
-
[(self.slug, t) for t in ours])
|
| 275 |
if tracking_error:
|
| 276 |
meta.update(trackingReady=False, trackingCause=tracking_error)
|
| 277 |
elif ours is not None:
|
|
@@ -532,17 +527,21 @@ def _top3(cur, terms=None):
|
|
| 532 |
return {}
|
| 533 |
cs = 'TRY_CAST("click_share" AS DOUBLE)' if "click_share" in cols else "NULL"
|
| 534 |
cv = 'TRY_CAST("conversion_share" AS DOUBLE)' if "conversion_share" in cols else "NULL"
|
| 535 |
-
pe = 'CAST("_period_end" AS
|
| 536 |
wanted = None if terms is None else sorted({_norm(t) for t in terms})
|
| 537 |
if wanted == []:
|
| 538 |
return {}
|
| 539 |
-
where = (f' AND {_term_sql("search_term")} IN (
|
| 540 |
if wanted is not None else '')
|
| 541 |
rows = cur.execute(
|
| 542 |
-
f'SELECT "search_term", CAST("_period_start" AS VARCHAR), {pe}, "clicked_asin", '
|
| 543 |
f'TRY_CAST("click_share_rank" AS INTEGER), {cs}, {cv} FROM {BA_TABLE} '
|
| 544 |
f'WHERE TRY_CAST("click_share_rank" AS INTEGER) BETWEEN 1 AND 3 '
|
| 545 |
-
f'AND ({_weekly_period_sql()}){where}', wanted
|
|
|
|
|
|
|
|
|
|
|
|
|
| 546 |
out = {}
|
| 547 |
for term, start, end, asin, rank, share, conv in rows:
|
| 548 |
key = _norm(term)
|
|
@@ -553,10 +552,52 @@ def _top3(cur, terms=None):
|
|
| 553 |
return out
|
| 554 |
|
| 555 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 556 |
def _periods(cur):
|
| 557 |
if not {"_period_start", "_period_end"} <= _ba_columns(cur):
|
| 558 |
return []
|
| 559 |
-
rows = cur.execute(f'SELECT DISTINCT CAST("_period_start" AS VARCHAR) FROM {BA_TABLE} '
|
| 560 |
f'WHERE {_weekly_period_sql()}').fetchall()
|
| 561 |
return sorted((str(r[0]) for r in rows if r[0] is not None), reverse=True)
|
| 562 |
|
|
@@ -600,9 +641,10 @@ def tracked_count(tenant_key, *, st=None):
|
|
| 600 |
def _brand_source_sql(cur):
|
| 601 |
"""Share one dated BA/listings join between membership and bulk click-share readers."""
|
| 602 |
need = {"search_term", "clicked_asin", "click_share_rank", "_period_start", "_period_end"}
|
| 603 |
-
|
|
|
|
| 604 |
raise SourceNotReady("Tracked terms need dated Brand Analytics and listing ASINs.")
|
| 605 |
-
if not
|
| 606 |
raise SourceNotReady(BA_WEEKLY_MISSING)
|
| 607 |
return (f"WITH weekly AS (SELECT * FROM {BA_TABLE} WHERE {_weekly_period_sql()}), "
|
| 608 |
f"own_asins AS (SELECT DISTINCT trim(asin1) AS asin FROM {LISTINGS_TABLE} "
|
|
@@ -639,8 +681,23 @@ def brand_share_source(cur, terms, start=None, end=None):
|
|
| 639 |
yield from batch
|
| 640 |
|
| 641 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 642 |
def seed_terms(tenant_key, *, st=None, cur=None):
|
| 643 |
-
"""
|
| 644 |
st = _rt(tenant_key, st)
|
| 645 |
index = _term_index(tenant_key, st)
|
| 646 |
|
|
@@ -650,22 +707,13 @@ def seed_terms(tenant_key, *, st=None, cur=None):
|
|
| 650 |
cause = str(exc) if isinstance(exc, _ReadFailed) else type(exc).__name__
|
| 651 |
return f"Tracked terms could not be checked ({cause}). The next Amazon sync will retry."
|
| 652 |
|
| 653 |
-
def distinct(reader):
|
| 654 |
-
if reader is None or "search_query_data_search_query" not in _columns(reader, SQP_TABLE):
|
| 655 |
-
if reader is not None and _columns(reader, BA_TABLE):
|
| 656 |
-
return set()
|
| 657 |
-
raise TermsNotReady(TERMS_NOT_READY)
|
| 658 |
-
return {r[0] for r in reader.execute(f'SELECT DISTINCT {_term_sql("search_query_data_search_query")} '
|
| 659 |
-
f'AS term FROM {SQP_TABLE} WHERE {_term_sql("search_query_data_search_query")} <> \'\'').fetchall()}
|
| 660 |
-
|
| 661 |
def source(reader):
|
| 662 |
try:
|
| 663 |
-
found =
|
| 664 |
-
except Exception
|
| 665 |
if index is None:
|
| 666 |
raise
|
| 667 |
-
|
| 668 |
-
found = set()
|
| 669 |
try:
|
| 670 |
ours = _tracked_terms_source(reader)
|
| 671 |
except Exception as exc: # noqa: BLE001
|
|
@@ -1050,6 +1098,21 @@ def _clear_error(doc):
|
|
| 1050 |
return doc
|
| 1051 |
|
| 1052 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1053 |
def compare(tenant_key, synced_tables=("brand_analytics",), *, st=None, now=None, quote_asins=None):
|
| 1054 |
"""Compare the newest held week with the week before and raise owned alerts. Returns a summary."""
|
| 1055 |
st = _rt(tenant_key, st)
|
|
@@ -1073,104 +1136,40 @@ def compare(tenant_key, synced_tables=("brand_analytics",), *, st=None, now=None
|
|
| 1073 |
terms = _comparison_terms(tenant_key, st)
|
| 1074 |
held = _top3(cur, terms)
|
| 1075 |
ours = _listing_asins(cur)
|
| 1076 |
-
weeks = _doc(st, KEY_WEEKS)
|
| 1077 |
newest = periods[0]
|
| 1078 |
if daily and weeks.get("last_compared") != newest:
|
| 1079 |
return {"state": "awaits_weekly_pull", "period": newest}
|
| 1080 |
retry = set(weeks.get("pending_quotes") or ()) & set(quote_asins or ()) if daily else None
|
| 1081 |
if daily and not retry:
|
| 1082 |
return {"state": "already_compared", "period": newest}
|
| 1083 |
-
if not daily and weeks.get("last_compared") == newest:
|
| 1084 |
-
# Every read this pass succeeded on a compared week, so a stored sync failure is superseded.
|
| 1085 |
-
if "last_error" in weeks:
|
| 1086 |
-
st.update(KEY_WEEKS, _clear_error)
|
| 1087 |
return {"state": "already_compared", "period": newest}
|
| 1088 |
previous = periods[1] if len(periods) > 1 else None
|
| 1089 |
conf = settings(tenant_key, st=st)
|
| 1090 |
|
| 1091 |
-
|
| 1092 |
-
want = set()
|
| 1093 |
-
for term in terms:
|
| 1094 |
-
want |= set(((held.get(term) or {}).get(newest) or {}).get("rows", {}))
|
| 1095 |
-
if our_for.get(term):
|
| 1096 |
-
want.add(our_for[term])
|
| 1097 |
quotes, quote_cause = _daily_quotes(tenant_key, st, want)
|
| 1098 |
current = (weeks.get("metrics") or {}).get(newest) or {}
|
| 1099 |
metrics = {a: current[a] if daily and a not in retry and a in current else
|
| 1100 |
_quote_metric(quotes.get(a), a in ours, quote_cause) for a in sorted(want)}
|
| 1101 |
prior = ((weeks.get("metrics") or {}).get(previous) or {}) if previous else {}
|
| 1102 |
|
| 1103 |
-
raised,
|
|
|
|
| 1104 |
for asin, metric in metrics.items():
|
| 1105 |
parts = list(dict.fromkeys(c for c in (metric.get("priceCause"), metric.get("rankCause")) if c))
|
| 1106 |
if metric.get("price") is not None:
|
| 1107 |
parts.insert(0, f"Price {metric['price']:,.2f} {metric.get('currency') or ''}.".strip())
|
| 1108 |
checks.append({"term": "", "asin": asin, "trigger": "Daily price", "periodStart": newest,
|
| 1109 |
"detail": metric, "cause": " ".join(parts)})
|
| 1110 |
-
|
| 1111 |
-
undercut = conf["undercut_pct"] / 100.0
|
| 1112 |
-
surge = conf["rank_surge_pct"] / 100.0
|
| 1113 |
-
for term in terms:
|
| 1114 |
-
now_week = (held.get(term) or {}).get(newest)
|
| 1115 |
-
was_week = (held.get(term) or {}).get(previous)
|
| 1116 |
-
if not now_week or not was_week:
|
| 1117 |
-
continue
|
| 1118 |
-
top_now, top_was = now_week["rows"], was_week["rows"]
|
| 1119 |
-
end = now_week["end"]
|
| 1120 |
-
|
| 1121 |
-
def found(asin, key, detail, cause=None):
|
| 1122 |
-
raised.append({"term": term, "asin": asin, "trigger": key, "periodStart": newest,
|
| 1123 |
-
"periodEnd": end, "previousPeriodStart": previous, "detail": detail,
|
| 1124 |
-
"cause": cause})
|
| 1125 |
-
|
| 1126 |
-
def asin_cause(asin):
|
| 1127 |
-
m = metrics.get(asin) or {}
|
| 1128 |
-
return " ".join(c for c in (m.get("priceCause"), m.get("rankCause")) if c) or None
|
| 1129 |
-
|
| 1130 |
-
for asin in sorted(set(top_now) - set(top_was)) if not daily else ():
|
| 1131 |
-
found(asin, "new_asin", {"rank": top_now[asin][0]}, asin_cause(asin))
|
| 1132 |
-
for asin in sorted((set(top_was) - set(top_now)) & ours) if not daily else ():
|
| 1133 |
-
found(asin, "lost_spot", {"previousRank": top_was[asin][0]})
|
| 1134 |
-
mine = our_for.get(term)
|
| 1135 |
-
mine_m = metrics.get(mine) or {}
|
| 1136 |
-
if mine is None and set(top_now) - ours:
|
| 1137 |
-
checks.append({"term": term, "asin": None, "trigger": TRIGGERS["undercut"],
|
| 1138 |
-
"periodStart": newest,
|
| 1139 |
-
"cause": "None of our listing ASINs is in this term's top 3, so there is "
|
| 1140 |
-
"no price of ours to compare."})
|
| 1141 |
-
for asin in sorted(set(top_now) - ours):
|
| 1142 |
-
m = metrics.get(asin) or {}
|
| 1143 |
-
if mine is None:
|
| 1144 |
-
pass
|
| 1145 |
-
elif mine_m.get("price") is None or m.get("price") is None:
|
| 1146 |
-
cause = " ".join(c for c in (mine_m.get("priceCause") and f"Our price: {mine_m['priceCause']}",
|
| 1147 |
-
m.get("priceCause")) if c)
|
| 1148 |
-
checks.append({"term": term, "asin": asin, "trigger": TRIGGERS["undercut"],
|
| 1149 |
-
"periodStart": newest, "cause": cause or "No price was read."})
|
| 1150 |
-
elif m["price"] <= mine_m["price"] * (1 - undercut) and (not daily or asin in retry or mine in retry):
|
| 1151 |
-
found(asin, "undercut", {"ourAsin": mine, "ourPrice": mine_m["price"],
|
| 1152 |
-
"price": m["price"],
|
| 1153 |
-
"percentUnder": round(100 * (1 - m["price"] / mine_m["price"]), 2)},
|
| 1154 |
-
asin_cause(asin))
|
| 1155 |
-
old = (prior.get(asin) or {})
|
| 1156 |
-
if m.get("rank") is None:
|
| 1157 |
-
checks.append({"term": term, "asin": asin, "trigger": TRIGGERS["rank_surge"],
|
| 1158 |
-
"periodStart": newest, "cause": m.get("rankCause") or "No sales rank was read."})
|
| 1159 |
-
elif old.get("rank") is None:
|
| 1160 |
-
if old.get("rankCause"):
|
| 1161 |
-
checks.append({"term": term, "asin": asin, "trigger": TRIGGERS["rank_surge"],
|
| 1162 |
-
"periodStart": newest,
|
| 1163 |
-
"cause": f"The week before: {old['rankCause']}"})
|
| 1164 |
-
elif m["rank"] <= old["rank"] * (1 - surge) and (not daily or asin in retry):
|
| 1165 |
-
found(asin, "rank_surge", {"rank": m["rank"], "previousRank": old["rank"]},
|
| 1166 |
-
asin_cause(asin))
|
| 1167 |
-
|
| 1168 |
created = _alert_index(tenant_key, st).record(raised, checks, newest, conf["defaultOwner"], now)
|
| 1169 |
|
| 1170 |
ready = previous is not None
|
| 1171 |
|
| 1172 |
def _weeks(doc):
|
| 1173 |
-
doc = doc if isinstance(doc, dict) else {}
|
| 1174 |
doc.setdefault("metrics", {})[newest] = metrics
|
| 1175 |
doc["pending_quotes"] = [asin for asin, metric in metrics.items()
|
| 1176 |
if metric.get("price") is None or metric.get("rank") is None]
|
|
@@ -1185,6 +1184,64 @@ def compare(tenant_key, synced_tables=("brand_analytics",), *, st=None, now=None
|
|
| 1185 |
"created": created, "checks": checks}
|
| 1186 |
|
| 1187 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1188 |
# --------------------------------------------------------------------------- alerts
|
| 1189 |
|
| 1190 |
class _AlertIndex(_DocumentIndex):
|
|
@@ -1395,20 +1452,22 @@ def update_alert(tenant_key, alert_id, *, owner=None, status=None, due=None, use
|
|
| 1395 |
|
| 1396 |
# --------------------------------------------------------------------------- term detail
|
| 1397 |
|
| 1398 |
-
def term_detail(tenant_key, term, *, st=None):
|
| 1399 |
"""One term's held periods, the newest top 3 with week-over-week change, and the click share trend."""
|
| 1400 |
st = _rt(tenant_key, st)
|
| 1401 |
key = _norm(term)
|
| 1402 |
-
with _reader(tenant_key, st) as cur:
|
| 1403 |
sqp = _sqp(cur, key) if cur is not None else []
|
| 1404 |
-
weeks_held = _top3(cur, [key]).get(key, {})
|
| 1405 |
candidates = {a for week in weeks_held.values() for a in week["rows"]}
|
| 1406 |
ours = _our_asins(cur, sqp, candidates) if cur is not None else set()
|
| 1407 |
periods = sorted(weeks_held, reverse=True)
|
| 1408 |
metric_asins = sorted(weeks_held[periods[0]]["rows"]) if periods else []
|
|
|
|
| 1409 |
state, *values = _paths(tenant_key, st, KEY_WEEKS, [("comparison",)] +
|
| 1410 |
-
[("metrics",
|
| 1411 |
-
metric =
|
|
|
|
| 1412 |
top3 = []
|
| 1413 |
if periods:
|
| 1414 |
newest = weeks_held[periods[0]]["rows"]
|
|
|
|
| 231 |
self._put_meta(con, meta)
|
| 232 |
|
| 233 |
def _insert(self, con, rows):
|
|
|
|
|
|
|
| 234 |
statement = self.term_sql("INSERT INTO {terms} (tenant, term, label, active, starred) "
|
| 235 |
+
"SELECT * FROM unnest(%s::text[], %s::text[], %s::text[], %s::boolean[], %s::boolean[]) "
|
| 236 |
+
"ON CONFLICT (tenant, term) DO NOTHING")
|
| 237 |
+
for start in range(0, len(rows), 1000):
|
| 238 |
+
con.execute(statement, [list(values) for values in zip(*rows[start:start + 1000])])
|
| 239 |
|
| 240 |
def seed(self, found, legacy, *, ours=None, tracking_error=None):
|
| 241 |
with self._locked(prepare=True) as (con, meta):
|
|
|
|
| 264 |
self._insert(con, [(self.slug, t, display_term(t), True, False) for t in found if t])
|
| 265 |
# Only a checked ownership result may replace membership, including a valid empty set.
|
| 266 |
if ours is not None:
|
|
|
|
|
|
|
| 267 |
self._insert(con, [(self.slug, t, display_term(t), True, False) for t in ours])
|
| 268 |
+
con.execute(self.term_sql("UPDATE {terms} SET ours = (term = ANY(%s)) WHERE tenant = %s "
|
| 269 |
+
"AND ours IS DISTINCT FROM (term = ANY(%s))"), [list(ours), self.slug, list(ours)])
|
|
|
|
|
|
|
| 270 |
if tracking_error:
|
| 271 |
meta.update(trackingReady=False, trackingCause=tracking_error)
|
| 272 |
elif ours is not None:
|
|
|
|
| 527 |
return {}
|
| 528 |
cs = 'TRY_CAST("click_share" AS DOUBLE)' if "click_share" in cols else "NULL"
|
| 529 |
cv = 'TRY_CAST("conversion_share" AS DOUBLE)' if "conversion_share" in cols else "NULL"
|
| 530 |
+
pe = 'CAST(TRY_CAST("_period_end" AS DATE) AS VARCHAR)'
|
| 531 |
wanted = None if terms is None else sorted({_norm(t) for t in terms})
|
| 532 |
if wanted == []:
|
| 533 |
return {}
|
| 534 |
+
where = (f' AND {_term_sql("search_term")} IN (SELECT unnest(?::VARCHAR[]))'
|
| 535 |
if wanted is not None else '')
|
| 536 |
rows = cur.execute(
|
| 537 |
+
f'SELECT "search_term", CAST(TRY_CAST("_period_start" AS DATE) AS VARCHAR), {pe}, "clicked_asin", '
|
| 538 |
f'TRY_CAST("click_share_rank" AS INTEGER), {cs}, {cv} FROM {BA_TABLE} '
|
| 539 |
f'WHERE TRY_CAST("click_share_rank" AS INTEGER) BETWEEN 1 AND 3 '
|
| 540 |
+
f'AND ({_weekly_period_sql()}){where}', [wanted] if wanted is not None else []).fetchall()
|
| 541 |
+
return _ranked_history(rows, wanted)
|
| 542 |
+
|
| 543 |
+
|
| 544 |
+
def _ranked_history(rows, wanted=None):
|
| 545 |
out = {}
|
| 546 |
for term, start, end, asin, rank, share, conv in rows:
|
| 547 |
key = _norm(term)
|
|
|
|
| 552 |
return out
|
| 553 |
|
| 554 |
|
| 555 |
+
def _page_history(cur, terms, selected):
|
| 556 |
+
"""Read page week counts and the selected term's ranks in one source scan."""
|
| 557 |
+
cols = _ba_columns(cur)
|
| 558 |
+
wanted = sorted(set(terms) | ({selected} if selected else set()))
|
| 559 |
+
if not wanted or not {"search_term", "_period_start", "_period_end"} <= cols:
|
| 560 |
+
return {}, {}
|
| 561 |
+
optional = {name: f'TRY_CAST("{name}" AS {kind})' if name in cols else f'NULL::{kind}'
|
| 562 |
+
for name, kind in (("clicked_asin", "VARCHAR"), ("click_share_rank", "INTEGER"),
|
| 563 |
+
("click_share", "DOUBLE"), ("conversion_share", "DOUBLE"))}
|
| 564 |
+
query = (f'WITH held AS MATERIALIZED (SELECT {_term_sql("search_term")} AS term, '
|
| 565 |
+
'CAST(TRY_CAST("_period_start" AS DATE) AS VARCHAR) AS start, '
|
| 566 |
+
'CAST(TRY_CAST("_period_end" AS DATE) AS VARCHAR) AS stop, '
|
| 567 |
+
+ ', '.join(f'{expr} AS "{name}"' for name, expr in optional.items()) +
|
| 568 |
+
f' FROM {BA_TABLE} WHERE {_weekly_period_sql()} '
|
| 569 |
+
f'AND {_term_sql("search_term")} IN (SELECT unnest(?::VARCHAR[]))) '
|
| 570 |
+
'SELECT term, COUNT(DISTINCT start) AS weeks, NULL AS start, NULL AS stop, '
|
| 571 |
+
'NULL AS asin, NULL::INTEGER AS rank, NULL::DOUBLE AS share, NULL::DOUBLE AS conversion '
|
| 572 |
+
'FROM held GROUP BY term UNION ALL '
|
| 573 |
+
'SELECT term, NULL, start, stop, clicked_asin, click_share_rank, click_share, conversion_share '
|
| 574 |
+
'FROM held WHERE term = ? AND click_share_rank BETWEEN 1 AND 3')
|
| 575 |
+
rows = cur.execute(query, [wanted, selected]).fetchall()
|
| 576 |
+
counts = {term: weeks for term, weeks, *_rest in rows if weeks is not None}
|
| 577 |
+
history = _ranked_history([(term, *values) for term, weeks, *values in rows if weeks is None])
|
| 578 |
+
return counts, history.get(selected, {})
|
| 579 |
+
|
| 580 |
+
|
| 581 |
+
def term_page_history(tenant_key, page, *, st=None):
|
| 582 |
+
"""Share the page's weekly scan with selected-term detail without preparing stored state."""
|
| 583 |
+
st = _rt(tenant_key, st)
|
| 584 |
+
selected = page["term"]
|
| 585 |
+
if not page["terms"] and not selected:
|
| 586 |
+
return {}, {}
|
| 587 |
+
try:
|
| 588 |
+
with _reader(tenant_key, st) as cur:
|
| 589 |
+
counts, history = _page_history(cur, page["terms"], selected)
|
| 590 |
+
detail = term_detail(tenant_key, selected, st=st, reader=cur, history=history) if selected else {}
|
| 591 |
+
except Exception as exc:
|
| 592 |
+
cause = str(exc) if isinstance(exc, _ReadFailed) else type(exc).__name__
|
| 593 |
+
raise SourceNotReady(cause, page=page) from exc
|
| 594 |
+
return counts, detail
|
| 595 |
+
|
| 596 |
+
|
| 597 |
def _periods(cur):
|
| 598 |
if not {"_period_start", "_period_end"} <= _ba_columns(cur):
|
| 599 |
return []
|
| 600 |
+
rows = cur.execute(f'SELECT DISTINCT CAST(TRY_CAST("_period_start" AS DATE) AS VARCHAR) FROM {BA_TABLE} '
|
| 601 |
f'WHERE {_weekly_period_sql()}').fetchall()
|
| 602 |
return sorted((str(r[0]) for r in rows if r[0] is not None), reverse=True)
|
| 603 |
|
|
|
|
| 641 |
def _brand_source_sql(cur):
|
| 642 |
"""Share one dated BA/listings join between membership and bulk click-share readers."""
|
| 643 |
need = {"search_term", "clicked_asin", "click_share_rank", "_period_start", "_period_end"}
|
| 644 |
+
cols, weekly = _ba_source(cur)
|
| 645 |
+
if cur is None or not need <= cols or "asin1" not in _columns(cur, LISTINGS_TABLE):
|
| 646 |
raise SourceNotReady("Tracked terms need dated Brand Analytics and listing ASINs.")
|
| 647 |
+
if not weekly:
|
| 648 |
raise SourceNotReady(BA_WEEKLY_MISSING)
|
| 649 |
return (f"WITH weekly AS (SELECT * FROM {BA_TABLE} WHERE {_weekly_period_sql()}), "
|
| 650 |
f"own_asins AS (SELECT DISTINCT trim(asin1) AS asin FROM {LISTINGS_TABLE} "
|
|
|
|
| 681 |
yield from batch
|
| 682 |
|
| 683 |
|
| 684 |
+
def _discover_terms(reader):
|
| 685 |
+
"""Union the two source vocabularies without treating SQP coverage as BA membership."""
|
| 686 |
+
if reader is None:
|
| 687 |
+
raise TermsNotReady(TERMS_NOT_READY)
|
| 688 |
+
sources = []
|
| 689 |
+
if "search_query_data_search_query" in _columns(reader, SQP_TABLE):
|
| 690 |
+
sources.append(f'SELECT {_term_sql("search_query_data_search_query")} AS term FROM {SQP_TABLE}')
|
| 691 |
+
if {"search_term", "_period_start", "_period_end"} <= _columns(reader, BA_TABLE):
|
| 692 |
+
sources.append(f'SELECT {_term_sql("search_term")} AS term FROM {BA_TABLE} WHERE {_weekly_period_sql()}')
|
| 693 |
+
if not sources:
|
| 694 |
+
raise TermsNotReady(TERMS_NOT_READY)
|
| 695 |
+
return {row[0] for row in reader.execute(
|
| 696 |
+
'SELECT DISTINCT term FROM (' + ' UNION ALL '.join(sources) + ") WHERE term <> ''").fetchall()}
|
| 697 |
+
|
| 698 |
+
|
| 699 |
def seed_terms(tenant_key, *, st=None, cur=None):
|
| 700 |
+
"""Discover SQP queries and held weekly BA terms while preserving authored membership."""
|
| 701 |
st = _rt(tenant_key, st)
|
| 702 |
index = _term_index(tenant_key, st)
|
| 703 |
|
|
|
|
| 707 |
cause = str(exc) if isinstance(exc, _ReadFailed) else type(exc).__name__
|
| 708 |
return f"Tracked terms could not be checked ({cause}). The next Amazon sync will retry."
|
| 709 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 710 |
def source(reader):
|
| 711 |
try:
|
| 712 |
+
found = _discover_terms(reader)
|
| 713 |
+
except Exception as exc:
|
| 714 |
if index is None:
|
| 715 |
raise
|
| 716 |
+
return set(), None, failure_cause(exc)
|
|
|
|
| 717 |
try:
|
| 718 |
ours = _tracked_terms_source(reader)
|
| 719 |
except Exception as exc: # noqa: BLE001
|
|
|
|
| 1098 |
return doc
|
| 1099 |
|
| 1100 |
|
| 1101 |
+
def _weekly_state(doc):
|
| 1102 |
+
"""Normalize legacy timestamp week keys before comparing with connector date keys."""
|
| 1103 |
+
out = dict(doc)
|
| 1104 |
+
if out.get("last_compared"):
|
| 1105 |
+
out["last_compared"] = dt.date.fromisoformat(str(out["last_compared"])[:10]).isoformat()
|
| 1106 |
+
metrics = {}
|
| 1107 |
+
for key, value in (out.get("metrics") or {}).items():
|
| 1108 |
+
day = dt.date.fromisoformat(str(key)[:10]).isoformat()
|
| 1109 |
+
if day in metrics and metrics[day] != value:
|
| 1110 |
+
raise SourceNotReady("Stored weekly metrics disagree for one week.")
|
| 1111 |
+
metrics[day] = value
|
| 1112 |
+
out["metrics"] = metrics
|
| 1113 |
+
return out
|
| 1114 |
+
|
| 1115 |
+
|
| 1116 |
def compare(tenant_key, synced_tables=("brand_analytics",), *, st=None, now=None, quote_asins=None):
|
| 1117 |
"""Compare the newest held week with the week before and raise owned alerts. Returns a summary."""
|
| 1118 |
st = _rt(tenant_key, st)
|
|
|
|
| 1136 |
terms = _comparison_terms(tenant_key, st)
|
| 1137 |
held = _top3(cur, terms)
|
| 1138 |
ours = _listing_asins(cur)
|
| 1139 |
+
weeks = _weekly_state(_doc(st, KEY_WEEKS))
|
| 1140 |
newest = periods[0]
|
| 1141 |
if daily and weeks.get("last_compared") != newest:
|
| 1142 |
return {"state": "awaits_weekly_pull", "period": newest}
|
| 1143 |
retry = set(weeks.get("pending_quotes") or ()) & set(quote_asins or ()) if daily else None
|
| 1144 |
if daily and not retry:
|
| 1145 |
return {"state": "already_compared", "period": newest}
|
| 1146 |
+
if not daily and weeks.get("last_compared") == newest and not weeks.get("last_error"):
|
|
|
|
|
|
|
|
|
|
| 1147 |
return {"state": "already_compared", "period": newest}
|
| 1148 |
previous = periods[1] if len(periods) > 1 else None
|
| 1149 |
conf = settings(tenant_key, st=st)
|
| 1150 |
|
| 1151 |
+
want = {asin for term in terms for asin in ((held.get(term) or {}).get(newest) or {}).get("rows", {})}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1152 |
quotes, quote_cause = _daily_quotes(tenant_key, st, want)
|
| 1153 |
current = (weeks.get("metrics") or {}).get(newest) or {}
|
| 1154 |
metrics = {a: current[a] if daily and a not in retry and a in current else
|
| 1155 |
_quote_metric(quotes.get(a), a in ours, quote_cause) for a in sorted(want)}
|
| 1156 |
prior = ((weeks.get("metrics") or {}).get(previous) or {}) if previous else {}
|
| 1157 |
|
| 1158 |
+
raised, term_checks = _term_signals(terms, held, periods, ours, metrics, prior, conf, retry)
|
| 1159 |
+
checks = []
|
| 1160 |
for asin, metric in metrics.items():
|
| 1161 |
parts = list(dict.fromkeys(c for c in (metric.get("priceCause"), metric.get("rankCause")) if c))
|
| 1162 |
if metric.get("price") is not None:
|
| 1163 |
parts.insert(0, f"Price {metric['price']:,.2f} {metric.get('currency') or ''}.".strip())
|
| 1164 |
checks.append({"term": "", "asin": asin, "trigger": "Daily price", "periodStart": newest,
|
| 1165 |
"detail": metric, "cause": " ".join(parts)})
|
| 1166 |
+
checks.extend(term_checks)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1167 |
created = _alert_index(tenant_key, st).record(raised, checks, newest, conf["defaultOwner"], now)
|
| 1168 |
|
| 1169 |
ready = previous is not None
|
| 1170 |
|
| 1171 |
def _weeks(doc):
|
| 1172 |
+
doc = _weekly_state(doc if isinstance(doc, dict) else {})
|
| 1173 |
doc.setdefault("metrics", {})[newest] = metrics
|
| 1174 |
doc["pending_quotes"] = [asin for asin, metric in metrics.items()
|
| 1175 |
if metric.get("price") is None or metric.get("rank") is None]
|
|
|
|
| 1184 |
"created": created, "checks": checks}
|
| 1185 |
|
| 1186 |
|
| 1187 |
+
def _term_signals(terms, held, periods, ours, metrics, prior, conf, retry):
|
| 1188 |
+
"""Derive weekly and quote-retry signals without changing alert state."""
|
| 1189 |
+
raised, checks = [], []
|
| 1190 |
+
if len(periods) < 2:
|
| 1191 |
+
return raised, checks
|
| 1192 |
+
newest, previous = periods[:2]
|
| 1193 |
+
daily = retry is not None
|
| 1194 |
+
undercut, surge = conf["undercut_pct"] / 100.0, conf["rank_surge_pct"] / 100.0
|
| 1195 |
+
for term in terms:
|
| 1196 |
+
now_week = (held.get(term) or {}).get(newest)
|
| 1197 |
+
was_week = (held.get(term) or {}).get(previous)
|
| 1198 |
+
if not now_week or not was_week:
|
| 1199 |
+
continue
|
| 1200 |
+
top_now, top_was = now_week["rows"], was_week["rows"]
|
| 1201 |
+
|
| 1202 |
+
def found(asin, key, detail, cause=None):
|
| 1203 |
+
raised.append({"term": term, "asin": asin, "trigger": key, "periodStart": newest,
|
| 1204 |
+
"periodEnd": now_week["end"], "previousPeriodStart": previous,
|
| 1205 |
+
"detail": detail, "cause": cause})
|
| 1206 |
+
|
| 1207 |
+
def asin_cause(asin):
|
| 1208 |
+
m = metrics.get(asin) or {}
|
| 1209 |
+
return " ".join(c for c in (m.get("priceCause"), m.get("rankCause")) if c) or None
|
| 1210 |
+
|
| 1211 |
+
for asin in sorted(set(top_now) - set(top_was)) if not daily else ():
|
| 1212 |
+
found(asin, "new_asin", {"rank": top_now[asin][0]}, asin_cause(asin))
|
| 1213 |
+
for asin in sorted((set(top_was) - set(top_now)) & ours) if not daily else ():
|
| 1214 |
+
found(asin, "lost_spot", {"previousRank": top_was[asin][0]})
|
| 1215 |
+
mine = _comparison_asin(now_week, ours)
|
| 1216 |
+
mine_m = metrics.get(mine) or {}
|
| 1217 |
+
if mine is None and set(top_now) - ours:
|
| 1218 |
+
checks.append({"term": term, "asin": None, "trigger": TRIGGERS["undercut"], "periodStart": newest,
|
| 1219 |
+
"cause": "None of our listing ASINs is in this term's top 3, so there is "
|
| 1220 |
+
"no price of ours to compare."})
|
| 1221 |
+
for asin in sorted(set(top_now) - ours):
|
| 1222 |
+
m = metrics.get(asin) or {}
|
| 1223 |
+
if mine is not None:
|
| 1224 |
+
if mine_m.get("price") is None or m.get("price") is None:
|
| 1225 |
+
cause = " ".join(c for c in (mine_m.get("priceCause") and f"Our price: {mine_m['priceCause']}",
|
| 1226 |
+
m.get("priceCause")) if c)
|
| 1227 |
+
checks.append({"term": term, "asin": asin, "trigger": TRIGGERS["undercut"],
|
| 1228 |
+
"periodStart": newest, "cause": cause or "No price was read."})
|
| 1229 |
+
elif m["price"] <= mine_m["price"] * (1 - undercut) and (not daily or asin in retry or mine in retry):
|
| 1230 |
+
found(asin, "undercut", {"ourAsin": mine, "ourPrice": mine_m["price"], "price": m["price"],
|
| 1231 |
+
"percentUnder": round(100 * (1 - m["price"] / mine_m["price"]), 2)}, asin_cause(asin))
|
| 1232 |
+
old = prior.get(asin) or {}
|
| 1233 |
+
if m.get("rank") is None:
|
| 1234 |
+
checks.append({"term": term, "asin": asin, "trigger": TRIGGERS["rank_surge"],
|
| 1235 |
+
"periodStart": newest, "cause": m.get("rankCause") or "No sales rank was read."})
|
| 1236 |
+
elif old.get("rank") is None:
|
| 1237 |
+
if old.get("rankCause"):
|
| 1238 |
+
checks.append({"term": term, "asin": asin, "trigger": TRIGGERS["rank_surge"],
|
| 1239 |
+
"periodStart": newest, "cause": f"The week before: {old['rankCause']}"})
|
| 1240 |
+
elif m["rank"] <= old["rank"] * (1 - surge) and (not daily or asin in retry):
|
| 1241 |
+
found(asin, "rank_surge", {"rank": m["rank"], "previousRank": old["rank"]}, asin_cause(asin))
|
| 1242 |
+
return raised, checks
|
| 1243 |
+
|
| 1244 |
+
|
| 1245 |
# --------------------------------------------------------------------------- alerts
|
| 1246 |
|
| 1247 |
class _AlertIndex(_DocumentIndex):
|
|
|
|
| 1452 |
|
| 1453 |
# --------------------------------------------------------------------------- term detail
|
| 1454 |
|
| 1455 |
+
def term_detail(tenant_key, term, *, st=None, reader=None, history=None):
|
| 1456 |
"""One term's held periods, the newest top 3 with week-over-week change, and the click share trend."""
|
| 1457 |
st = _rt(tenant_key, st)
|
| 1458 |
key = _norm(term)
|
| 1459 |
+
with contextlib.nullcontext(reader) if reader is not None else _reader(tenant_key, st) as cur:
|
| 1460 |
sqp = _sqp(cur, key) if cur is not None else []
|
| 1461 |
+
weeks_held = _top3(cur, [key]).get(key, {}) if history is None else history
|
| 1462 |
candidates = {a for week in weeks_held.values() for a in week["rows"]}
|
| 1463 |
ours = _our_asins(cur, sqp, candidates) if cur is not None else set()
|
| 1464 |
periods = sorted(weeks_held, reverse=True)
|
| 1465 |
metric_asins = sorted(weeks_held[periods[0]]["rows"]) if periods else []
|
| 1466 |
+
variants = (periods[0], periods[0] + "T00:00:00Z", periods[0] + " 00:00:00") if periods else ()
|
| 1467 |
state, *values = _paths(tenant_key, st, KEY_WEEKS, [("comparison",)] +
|
| 1468 |
+
[("metrics", week, asin) for asin in metric_asins for week in variants])
|
| 1469 |
+
metric = {asin: next((value for value in values[i * 3:i * 3 + 3] if value is not None), None)
|
| 1470 |
+
for i, asin in enumerate(metric_asins)}
|
| 1471 |
top3 = []
|
| 1472 |
if periods:
|
| 1473 |
newest = weeks_held[periods[0]]["rows"]
|
web/public/AIOS_DEPLOY_RECEIPT
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
|
|
|
|
| 1 |
+
bb954b673746f91c1826c1869e08268a546c64e816c8669fbb9654a6ee4bb30d
|