fsanyoto commited on
Commit
f694a2b
·
verified ·
1 Parent(s): 82ff59c

Deploy AIOS web (React glide grid + FastAPI slice)

Browse files
RELEASES.json CHANGED
@@ -1,4 +1,4 @@
1
  {
2
- "current": "99ba051e73",
3
  "releases": []
4
  }
 
1
  {
2
+ "current": "2f4d90c03d",
3
  "releases": []
4
  }
VERSION CHANGED
@@ -1 +1 @@
1
- 99ba051e73
 
1
+ 2f4d90c03d
api/amazon_relational.py CHANGED
@@ -994,6 +994,7 @@ def publication_sources(cur, keys):
994
  if schema is None:
995
  continue
996
  spec = grid_source(cur, key, allow_lookups=False, status=None, schema=schema)
 
997
  if amazon_schema.CONNECTION_ID in dict(schema[0]):
998
  spec['cols'][amazon_schema.CONNECTION_ID] = ('"_connection_id"', _text)
999
  result[key] = spec
 
994
  if schema is None:
995
  continue
996
  spec = grid_source(cur, key, allow_lookups=False, status=None, schema=schema)
997
+ spec['observation_tables'] = [SOURCE[key]]
998
  if amazon_schema.CONNECTION_ID in dict(schema[0]):
999
  spec['cols'][amazon_schema.CONNECTION_ID] = ('"_connection_id"', _text)
1000
  result[key] = spec
api/routes_competitor_watch.py CHANGED
@@ -217,21 +217,13 @@ def _not_ready_message(cw, periods):
217
  return cw.NOT_READY
218
 
219
 
220
- def _term_rows(cw, tenant, st, page):
221
  """Read dated-week counts for this server page without loading its ranking history."""
222
  terms = page["terms"]
223
- counts = {}
224
- if terms:
225
  try:
226
  with cw._reader(tenant, st) as cur:
227
- cols = cw._ba_columns(cur)
228
- if {"search_term", "_period_start", "_period_end"} <= cols:
229
- key = cw._term_sql("search_term")
230
- rows = cur.execute(
231
- f'SELECT {key}, COUNT(DISTINCT CAST("_period_start" AS DATE)) '
232
- f'FROM {cw.BA_TABLE} WHERE {key} IN (' + ','.join('?' for _ in terms) + ') '
233
- f'AND ({cw._weekly_period_sql()}) GROUP BY {key}', terms).fetchall()
234
- counts = dict(rows)
235
  except Exception as exc: # noqa: BLE001
236
  cause = str(exc) if isinstance(exc, cw._ReadFailed) else type(exc).__name__
237
  raise cw.SourceNotReady(cause, page=page) from exc
@@ -315,6 +307,6 @@ def app_screen(app: str = Query(..., min_length=1), term: str | None = Query(Non
315
  page = _call(cw.term_page, tenant, st=session.runtime, search=q, offset=offset,
316
  limit=limit, selected=term, tracked=tracked, termOrder=termOrder)
317
  terms, picked = page["terms"], page["term"]
318
- page = {**page, "termRows": _call(_term_rows, cw, tenant, session.runtime, page)}
319
- detail = _call(cw.term_detail, tenant, picked, st=session.runtime) if picked else {}
320
  return screen_payload(cw, detail, terms, picked, _may_edit(session, record), page=page)
 
217
  return cw.NOT_READY
218
 
219
 
220
+ def _term_rows(cw, tenant, st, page, counts=None):
221
  """Read dated-week counts for this server page without loading its ranking history."""
222
  terms = page["terms"]
223
+ if counts is None:
 
224
  try:
225
  with cw._reader(tenant, st) as cur:
226
+ counts, _history = cw._page_history(cur, terms, None)
 
 
 
 
 
 
 
227
  except Exception as exc: # noqa: BLE001
228
  cause = str(exc) if isinstance(exc, cw._ReadFailed) else type(exc).__name__
229
  raise cw.SourceNotReady(cause, page=page) from exc
 
307
  page = _call(cw.term_page, tenant, st=session.runtime, search=q, offset=offset,
308
  limit=limit, selected=term, tracked=tracked, termOrder=termOrder)
309
  terms, picked = page["terms"], page["term"]
310
+ counts, detail = _call(cw.term_page_history, tenant, page, st=session.runtime)
311
+ page = {**page, "termRows": _term_rows(cw, tenant, session.runtime, page, counts)}
312
  return screen_payload(cw, detail, terms, picked, _may_edit(session, record), page=page)
platform/core/received_changes.py CHANGED
@@ -24,6 +24,7 @@ PROJECTION_VERSION = 1
24
  IMAGE_DIGEST = "md5_number(image)"
25
  _LISTENERS = []
26
  _PROJECTIONS = {}
 
27
  _LOCK = threading.RLock()
28
  _LOG = logging.getLogger(__name__)
29
 
@@ -313,6 +314,8 @@ def _write_blob(rt, path, data):
313
  """Reuse immutable content and verify persistence before publishing its reference."""
314
  from core import blob_envelope as be
315
  digest, size = _digest(data), len(data)
 
 
316
  raw = rt.download_bytes(path)
317
  if raw is None:
318
  stored = be.seal(rt.key, path, data)
@@ -324,18 +327,45 @@ def _write_blob(rt, path, data):
324
  return {"sha256": digest, "bytes": size}
325
 
326
 
327
- def _persist_baseline(rt, identity, binding, path, snapshot, stamp):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
328
  from core import blob_envelope as be
329
  parts, content, size = [], hashlib.sha256(), 0
 
330
  with path.open("rb") as handle:
331
  while chunk := handle.read(BASELINE_CHUNK_BYTES - len(be.HEADER) - 28):
332
- parts.append(_write_blob(rt, _blob_path(identity, "chunks", _digest(chunk)), chunk))
 
 
333
  content.update(chunk)
334
  size += len(chunk)
335
- manifest = {**binding, "snapshot": snapshot, "at": stamp, "parts": parts,
336
  "sha256": content.hexdigest(), "bytes": size}
337
  data = _json_bytes(manifest)
338
- return _write_blob(rt, _blob_path(identity, "manifests", _digest(data)), data)
339
 
340
 
341
  def _cached_baseline(cache, identity, reference):
@@ -348,6 +378,7 @@ def _cache_baseline(cache, identity, reference, path):
348
  for stale in cache.glob(identity + "-*.parquet"):
349
  stale.unlink(missing_ok=True)
350
  os.replace(path, _cached_baseline(cache, identity, reference))
 
351
 
352
 
353
  def _restore_baseline(rt, identity, binding, reference, path, con, cache):
@@ -355,11 +386,21 @@ def _restore_baseline(rt, identity, binding, reference, path, con, cache):
355
  data = _read_blob(rt, _blob_path(identity, "manifests", reference["sha256"]),
356
  reference["sha256"], reference["bytes"])
357
  manifest = json.loads(data)
358
- if (any(manifest.get(key) != value for key, value in binding.items())
 
359
  or not isinstance(manifest.get("snapshot"), str) or not manifest["snapshot"]
360
  or not isinstance(manifest.get("parts"), list) or not manifest["parts"]
361
  or not manifest.get("at")):
362
  raise BaselineUnavailable("The observation manifest has a different identity or shape.")
 
 
 
 
 
 
 
 
 
363
  _at(manifest["at"])
364
  cached = _cached_baseline(cache, identity, reference)
365
  if cached.exists():
@@ -374,7 +415,8 @@ def _restore_baseline(rt, identity, binding, reference, path, con, cache):
374
  for part in manifest["parts"]))
375
  if content != manifest["sha256"] or size != manifest["bytes"]:
376
  raise BaselineUnavailable("The observation snapshot failed content verification.")
377
- _validate_projection(con, path)
 
378
  return manifest
379
  except BaselineUnavailable:
380
  raise
@@ -428,8 +470,8 @@ def _project(con, query, path, database):
428
  raw.unlink()
429
 
430
 
431
- def _observation_state(rt, identity):
432
- document = _document(rt, rt.get(STORE_KEY, fresh=True))
433
  states = document.get("observations", {})
434
  if not isinstance(states, dict):
435
  raise BaselineUnavailable("The observation index is invalid.")
@@ -463,6 +505,7 @@ def _move_baseline(rt, identity, expected, updated, observed_at=None):
463
  result = rt.update(STORE_KEY, move, flush="sync")
464
  if result is None or result is False:
465
  raise BaselineUnavailable("The observation reference was not saved.")
 
466
 
467
 
468
  @contextmanager
@@ -485,14 +528,16 @@ def _compare_baseline(rt, source, database, spec, con, baseline, current, snapsh
485
  changed = (f"SELECT coalesce(b.row_id, a.row_id) FROM {digests.format('$1')} b FULL OUTER JOIN "
486
  f"{digests.format('$2')} a USING (row_id) WHERE b.image IS DISTINCT FROM a.image")
487
  files = [str(baseline), str(current)]
488
- ids = [row[0] for row in con.execute(changed + f" LIMIT {MAX_EVENTS + 1}", files).fetchall()]
 
489
  # A diff wider than the retained window cannot be replayed, so it is recorded as one bulk change.
490
- if len(ids) > MAX_EVENTS:
491
- count = con.execute(f"SELECT count(*) FROM ({changed})", files).fetchone()[0]
492
  print(f"[received-changes] {database}: {count} changed rows exceed the journal's "
493
  f"{MAX_EVENTS}-event window; recorded as a bulk change, rows not journaled", flush=True)
494
  return
495
- count, cursor, done = len(ids), con.cursor(), 0
 
 
496
  try:
497
  with _query_watchdog(cursor, deadline):
498
  picked = "(SELECT * FROM read_parquet({}) WHERE row_id IN (SELECT unnest($3::VARCHAR[])))"
@@ -527,7 +572,40 @@ def _observation_identity(rt, source, database):
527
  return hashlib.sha256(json.dumps([_tenant(rt), source, database]).encode()).hexdigest()
528
 
529
 
530
- def _snapshot(rt, source, database, spec, con, root, manifest, deadline=None):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
531
  """Compare verified durable observations in batches, recovering interrupted comparisons first."""
532
  if not spec.get("cols"):
533
  return
@@ -540,7 +618,7 @@ def _snapshot(rt, source, database, spec, con, root, manifest, deadline=None):
540
  if not stamp:
541
  raise BaselineUnavailable("Connector observations require a completed publication time.")
542
  stamp = _at(stamp)
543
- state, document = _observation_state(rt, identity)
544
  if state is None and (any((root / (identity + suffix)).exists() for suffix in (".parquet", ".json"))
545
  or any(event["source"] == source and event["database"] == database
546
  for event in document["events"])):
@@ -549,46 +627,105 @@ def _snapshot(rt, source, database, spec, con, root, manifest, deadline=None):
549
  with tempfile.TemporaryDirectory(dir=root, prefix="projection-") as directory:
550
  baseline, current = Path(directory) / "baseline.parquet", Path(directory) / "current.parquet"
551
 
552
- def complete(pending, observed=None):
553
- # A pending reference this pass wrote is already on disk; one from an earlier pass is restored.
554
- if observed is None:
555
- observed = _restore_baseline(rt, identity, binding, pending["pending"], current, con,
556
- cache)["snapshot"]
557
- if pending["current"]:
558
- _restore_baseline(rt, identity, binding, pending["current"], baseline, con, cache)
559
- if pending["current"]:
560
- _compare_baseline(rt, source, database, spec, con, baseline, current, observed, deadline)
561
- committed = {"current": pending["pending"], "pending": None}
562
- _move_baseline(rt, identity, pending, committed, observed_at=_at())
563
- _cache_baseline(cache, identity, pending["pending"], current)
564
- return committed
565
-
566
- if state and state["pending"]:
567
- state = complete(state)
568
- if state:
569
- prior = _restore_baseline(rt, identity, binding, state["current"], baseline, con, cache)
570
  prior_at = dt.datetime.fromisoformat(prior["at"].replace("Z", "+00:00"))
571
  incoming_at = dt.datetime.fromisoformat(stamp.replace("Z", "+00:00"))
572
- if prior["snapshot"] == snapshot or incoming_at < prior_at:
 
 
573
  return
574
- if incoming_at == prior_at:
575
  raise BaselineUnavailable("Distinct connector snapshots have ambiguous publication order.")
576
- expressions = ", ".join(f"{_quoted(key)} := {value[0]}" for key, value in spec["cols"].items())
577
- where = " WHERE " + spec["where"] if spec.get("where") else ""
578
- query = (f"SELECT CAST(({spec['id']}) AS VARCHAR) AS row_id, "
579
- f"CAST(to_json(struct_pack({expressions})) AS VARCHAR) AS image "
580
- f"FROM {spec.get('from_sql') or spec['table']}{where}")
581
- current.unlink(missing_ok=True)
582
- _project(con, query, current, database)
583
- reference = _persist_baseline(rt, identity, binding, current, snapshot, stamp)
584
- pending = {"current": state["current"] if state else None, "pending": reference}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
585
  _move_baseline(rt, identity, state, pending)
586
- complete(pending, snapshot)
 
 
 
 
 
 
587
 
588
 
589
- def _publication_order(rt, connector, specs, paths):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
590
  """Least recently observed databases first, then the smallest, so a deadline defers in rotation."""
591
- observed = _document(rt, rt.get(STORE_KEY, fresh=True)).get("observed") or {}
 
592
 
593
  def priority(item):
594
  database, spec = item
@@ -598,13 +735,17 @@ def _publication_order(rt, connector, specs, paths):
598
  return sorted(specs.items(), key=priority)
599
 
600
 
601
- def _project_database(rt, connector, database, spec, con, root, manifest, deadline):
602
  """Return None when the database committed, else the named reason it was deferred."""
603
  import duckdb
604
  from core import request_timing
 
605
  try:
606
  with _query_watchdog(con, deadline):
607
- _snapshot(rt, connector, database, spec, con, root, manifest, deadline)
 
 
 
608
  except request_timing.RequestDeadlineExceeded as exc:
609
  # The watchdog raises on exit after a clean body too; no cause means the database committed.
610
  return None if exc.__cause__ is None else "journal deadline passed"
@@ -613,6 +754,8 @@ def _project_database(rt, connector, database, spec, con, root, manifest, deadli
613
  except duckdb.Error as exc:
614
  # One database's failed query defers it alone; its stored baseline stays for the next pass.
615
  return type(exc).__name__
 
 
616
  return None
617
 
618
 
@@ -643,13 +786,14 @@ def mirror_hook(connector, tenant_key, manifest, _state_rows, paths):
643
  resolution_seconds = time.monotonic() - started
644
  snapshots_started = time.monotonic()
645
  completed = deferred = skipped = 0
646
- for database, spec in _publication_order(rt, connector, specs, paths):
 
647
  # A skipped database keeps its stored baseline and leads the next publication.
648
  if time.monotonic() >= deadline[0]:
649
  skipped += 1
650
  print(f"[received-changes] {database}: skipped, journal deadline passed", flush=True)
651
  continue
652
- reason = _project_database(rt, connector, str(database), spec, con, root, manifest, deadline)
653
  if reason is None:
654
  completed += 1
655
  continue
 
24
  IMAGE_DIGEST = "md5_number(image)"
25
  _LISTENERS = []
26
  _PROJECTIONS = {}
27
+ _VERIFIED_BASELINES = {}
28
  _LOCK = threading.RLock()
29
  _LOG = logging.getLogger(__name__)
30
 
 
314
  """Reuse immutable content and verify persistence before publishing its reference."""
315
  from core import blob_envelope as be
316
  digest, size = _digest(data), len(data)
317
+ if _insert_blob(rt, path, data):
318
+ return {"sha256": digest, "bytes": size}
319
  raw = rt.download_bytes(path)
320
  if raw is None:
321
  stored = be.seal(rt.key, path, data)
 
327
  return {"sha256": digest, "bytes": size}
328
 
329
 
330
+ def _insert_blob(rt, path, data):
331
+ """Verify a new Postgres blob in its insert transaction without transferring it back."""
332
+ from core import blob_envelope as be, store_pg
333
+ backend = rt._document_store() if hasattr(rt, "_document_store") else rt
334
+ if not isinstance(backend, store_pg.PgStore):
335
+ return False
336
+ if backend.tenant_slug != _tenant(rt):
337
+ raise BaselineUnavailable("The observation store belongs to another tenant.")
338
+ store_pg._effective_slug(backend.tenant_slug)
339
+ store_pg.check_write(backend.tenant_slug, "write received-change observation")
340
+ stored = be.seal(rt.key, path, data)
341
+ if len(stored) > BASELINE_CHUNK_BYTES:
342
+ raise BaselineUnavailable("The observation blob exceeds the store chunk size.")
343
+ name = rt.store_key(path) if hasattr(rt, "store_key") else path
344
+ with store_pg._pool().connection() as con:
345
+ row = con.execute(store_pg._q(
346
+ "INSERT INTO {tbl} (path, bytes) VALUES (%s, %s) ON CONFLICT (path) DO NOTHING "
347
+ "RETURNING octet_length(bytes), encode(sha256(bytes), 'hex')",
348
+ "store_blobs", backend.tenant_slug), [name, stored]).fetchone()
349
+ if row is not None and tuple(row) != (len(stored), _digest(stored)):
350
+ raise BaselineUnavailable("The observation upload failed content verification.")
351
+ return row is not None
352
+
353
+
354
+ def _persist_baseline(rt, identity, binding, path, snapshot, stamp, prior=None, **metadata):
355
  from core import blob_envelope as be
356
  parts, content, size = [], hashlib.sha256(), 0
357
+ verified = {(part["sha256"], part["bytes"]) for part in (prior or {}).get("parts", [])}
358
  with path.open("rb") as handle:
359
  while chunk := handle.read(BASELINE_CHUNK_BYTES - len(be.HEADER) - 28):
360
+ digest = _digest(chunk)
361
+ parts.append({"sha256": digest, "bytes": len(chunk)} if (digest, len(chunk)) in verified else
362
+ _write_blob(rt, _blob_path(identity, "chunks", digest), chunk))
363
  content.update(chunk)
364
  size += len(chunk)
365
+ manifest = {**binding, **metadata, "snapshot": snapshot, "at": stamp, "parts": parts,
366
  "sha256": content.hexdigest(), "bytes": size}
367
  data = _json_bytes(manifest)
368
+ return _write_blob(rt, _blob_path(identity, "manifests", _digest(data)), data), manifest
369
 
370
 
371
  def _cached_baseline(cache, identity, reference):
 
378
  for stale in cache.glob(identity + "-*.parquet"):
379
  stale.unlink(missing_ok=True)
380
  os.replace(path, _cached_baseline(cache, identity, reference))
381
+ _VERIFIED_BASELINES[(str(cache), identity)] = reference["sha256"]
382
 
383
 
384
  def _restore_baseline(rt, identity, binding, reference, path, con, cache):
 
386
  data = _read_blob(rt, _blob_path(identity, "manifests", reference["sha256"]),
387
  reference["sha256"], reference["bytes"])
388
  manifest = json.loads(data)
389
+ if (any(manifest.get(key) != value for key, value in binding.items() if key != "projection")
390
+ or not isinstance(manifest.get("projection"), str)
391
  or not isinstance(manifest.get("snapshot"), str) or not manifest["snapshot"]
392
  or not isinstance(manifest.get("parts"), list) or not manifest["parts"]
393
  or not manifest.get("at")):
394
  raise BaselineUnavailable("The observation manifest has a different identity or shape.")
395
+ _blob_path(identity, "projections", manifest["projection"])
396
+ for part in manifest["parts"]:
397
+ if (set(part) != {"sha256", "bytes"} or type(part["bytes"]) is not int
398
+ or not 0 < part["bytes"] <= BASELINE_CHUNK_BYTES):
399
+ raise BaselineUnavailable("The observation chunk reference is invalid.")
400
+ _blob_path(identity, "chunks", part["sha256"])
401
+ if (type(manifest.get("bytes")) is not int or manifest["bytes"] <= 0
402
+ or sum(part["bytes"] for part in manifest["parts"]) != manifest["bytes"]):
403
+ raise BaselineUnavailable("The observation content length is invalid.")
404
  _at(manifest["at"])
405
  cached = _cached_baseline(cache, identity, reference)
406
  if cached.exists():
 
415
  for part in manifest["parts"]))
416
  if content != manifest["sha256"] or size != manifest["bytes"]:
417
  raise BaselineUnavailable("The observation snapshot failed content verification.")
418
+ if _VERIFIED_BASELINES.get((str(cache), identity)) != reference["sha256"]:
419
+ _validate_projection(con, path)
420
  return manifest
421
  except BaselineUnavailable:
422
  raise
 
470
  raw.unlink()
471
 
472
 
473
+ def _observation_state(rt, identity, document=None):
474
+ document = _document(rt, rt.get(STORE_KEY, fresh=True)) if document is None else document
475
  states = document.get("observations", {})
476
  if not isinstance(states, dict):
477
  raise BaselineUnavailable("The observation index is invalid.")
 
505
  result = rt.update(STORE_KEY, move, flush="sync")
506
  if result is None or result is False:
507
  raise BaselineUnavailable("The observation reference was not saved.")
508
+ return result
509
 
510
 
511
  @contextmanager
 
528
  changed = (f"SELECT coalesce(b.row_id, a.row_id) FROM {digests.format('$1')} b FULL OUTER JOIN "
529
  f"{digests.format('$2')} a USING (row_id) WHERE b.image IS DISTINCT FROM a.image")
530
  files = [str(baseline), str(current)]
531
+ rows = con.execute(f"SELECT count(*) OVER (), * FROM ({changed}) LIMIT {MAX_EVENTS + 1}", files).fetchall()
532
+ count, ids = (rows[0][0] if rows else 0), [row[1] for row in rows]
533
  # A diff wider than the retained window cannot be replayed, so it is recorded as one bulk change.
534
+ if count > MAX_EVENTS:
 
535
  print(f"[received-changes] {database}: {count} changed rows exceed the journal's "
536
  f"{MAX_EVENTS}-event window; recorded as a bulk change, rows not journaled", flush=True)
537
  return
538
+ if not ids:
539
+ return
540
+ cursor, done = con.cursor(), 0
541
  try:
542
  with _query_watchdog(cursor, deadline):
543
  picked = "(SELECT * FROM read_parquet({}) WHERE row_id IN (SELECT unnest($3::VARCHAR[])))"
 
572
  return hashlib.sha256(json.dumps([_tenant(rt), source, database]).encode()).hexdigest()
573
 
574
 
575
+ def _complete_baseline(rt, identity, state, current, baseline, con, cache,
576
+ source, database, spec, deadline, observed, expected, unchanged=False, reset=False):
577
+ if state["current"] and not unchanged:
578
+ _compare_baseline(rt, source, database, spec, con, baseline, current, observed, deadline)
579
+ committed = {"current": state["pending"], "pending": None}
580
+ document = _move_baseline(rt, identity, expected, committed, observed_at=_at())
581
+ _cache_baseline(cache, identity, state["pending"], current)
582
+ if reset:
583
+ print(f"[received-changes] {database}: projection binding changed; new baseline, no events", flush=True)
584
+ return committed, document
585
+
586
+
587
+ def _prior_observations(rt, identity, binding, state, baseline, current, con, cache):
588
+ """Verify both references before recovering or replacing a changed projection."""
589
+ held = {}
590
+ for key, path in (("current", baseline), ("pending", current)):
591
+ if state and state[key]:
592
+ held[key] = _restore_baseline(rt, identity, binding, state[key], path, con, cache)
593
+ if len(held) == 2:
594
+ before, after = held["current"], held["pending"]
595
+ if dt.datetime.fromisoformat(after["at"].replace("Z", "+00:00")) <= dt.datetime.fromisoformat(
596
+ before["at"].replace("Z", "+00:00")):
597
+ raise BaselineUnavailable("The pending observation has invalid publication order.")
598
+ return held
599
+
600
+
601
+ def _refresh_observation(rt, identity, binding, state, prior, snapshot, stamp):
602
+ """Advance an identical input's publication order while retaining its verified content."""
603
+ data = _json_bytes({**prior, **binding, "snapshot": snapshot, "at": stamp})
604
+ reference = _write_blob(rt, _blob_path(identity, "manifests", _digest(data)), data)
605
+ return _move_baseline(rt, identity, state, {"current": reference, "pending": None}, observed_at=_at())
606
+
607
+
608
+ def _snapshot(rt, source, database, spec, con, root, manifest, deadline=None, publication=None):
609
  """Compare verified durable observations in batches, recovering interrupted comparisons first."""
610
  if not spec.get("cols"):
611
  return
 
618
  if not stamp:
619
  raise BaselineUnavailable("Connector observations require a completed publication time.")
620
  stamp = _at(stamp)
621
+ state, document = _observation_state(rt, identity, publication.document if publication else None)
622
  if state is None and (any((root / (identity + suffix)).exists() for suffix in (".parquet", ".json"))
623
  or any(event["source"] == source and event["database"] == database
624
  for event in document["events"])):
 
627
  with tempfile.TemporaryDirectory(dir=root, prefix="projection-") as directory:
628
  baseline, current = Path(directory) / "baseline.parquet", Path(directory) / "current.parquet"
629
 
630
+ restoring = time.monotonic()
631
+ held = _prior_observations(rt, identity, binding, state, baseline, current, con, cache)
632
+ print(f"[received-changes] {database}: restore_verify={time.monotonic() - restoring:.3f}s", flush=True)
633
+ reset = any(prior["projection"] != binding["projection"] for prior in held.values())
634
+ prior = held.get("pending") or held.get("current")
635
+ if state and state["pending"] and not reset:
636
+ state, document = _complete_baseline(rt, identity, state, current, baseline, con,
637
+ cache, source, database, spec, deadline, prior["snapshot"],
638
+ expected=state,
639
+ reset=bool(prior.get("reset_from")) and not state["current"])
640
+ # The recovered pending file became the cache, and is the next comparison's baseline.
641
+ baseline.unlink(missing_ok=True)
642
+ baseline.hardlink_to(_cached_baseline(cache, identity, state["current"]))
643
+ if prior:
 
 
 
 
644
  prior_at = dt.datetime.fromisoformat(prior["at"].replace("Z", "+00:00"))
645
  incoming_at = dt.datetime.fromisoformat(stamp.replace("Z", "+00:00"))
646
+ if incoming_at < prior_at:
647
+ if publication:
648
+ publication.document = document
649
  return
650
+ if incoming_at == prior_at and prior["snapshot"] != snapshot:
651
  raise BaselineUnavailable("Distinct connector snapshots have ambiguous publication order.")
652
+ if prior and not reset and prior["snapshot"] == snapshot:
653
+ if publication:
654
+ publication.document = document
655
+ return
656
+ if prior and not reset and spec.get("input_digest") == prior.get("input") and spec.get("input_digest"):
657
+ document = _refresh_observation(rt, identity, binding, state, prior, snapshot, stamp)
658
+ _cache_baseline(cache, identity, document["observations"][identity]["current"], baseline)
659
+ else:
660
+ document = _new_observation(rt, identity, binding, state, spec, con, current, baseline,
661
+ cache, source, database, snapshot, stamp, deadline, reset, prior)
662
+ if publication:
663
+ publication.document = document
664
+
665
+
666
+ def _new_observation(rt, identity, binding, state, spec, con, current, baseline, cache,
667
+ source, database, snapshot, stamp, deadline, reset, prior):
668
+ expressions = ", ".join(f"{_quoted(key)} := {value[0]}" for key, value in spec["cols"].items())
669
+ where = " WHERE " + spec["where"] if spec.get("where") else ""
670
+ query = (f"SELECT CAST(({spec['id']}) AS VARCHAR) AS row_id, "
671
+ f"CAST(to_json(struct_pack({expressions})) AS VARCHAR) AS image "
672
+ f"FROM {spec.get('from_sql') or spec['table']}{where}")
673
+ current.unlink(missing_ok=True)
674
+ started = time.monotonic()
675
+ _project(con, query, current, database)
676
+ projected = time.monotonic()
677
+ reference, observed = _persist_baseline(rt, identity, binding, current, snapshot, stamp,
678
+ prior=prior, input=spec.get("input_digest"), reset_from=prior["projection"] if reset else None)
679
+ persisted = time.monotonic()
680
+ pending = {"current": state["current"] if state and not reset else None, "pending": reference}
681
+ unchanged = bool(prior and prior["sha256"] == observed["sha256"])
682
+ expected = state
683
+ # Only a comparison that can emit events needs a durable pending checkpoint.
684
+ if pending["current"] and not unchanged:
685
  _move_baseline(rt, identity, state, pending)
686
+ expected = pending
687
+ _committed, document = _complete_baseline(rt, identity, pending, current, baseline,
688
+ con, cache, source, database, spec, deadline, snapshot, expected=expected,
689
+ unchanged=unchanged, reset=reset)
690
+ print(f"[received-changes] {database}: project={projected - started:.3f}s "
691
+ f"persist={persisted - projected:.3f}s compare_commit={time.monotonic() - persisted:.3f}s", flush=True)
692
+ return document
693
 
694
 
695
+ class _ObservationPublication:
696
+ """Share one authoritative journal read and ordered input digests across a publication."""
697
+
698
+ def __init__(self, rt, paths):
699
+ started = time.monotonic()
700
+ self.document = _document(rt, rt.get(STORE_KEY, fresh=True))
701
+ self.paths, self.digests = paths, {}
702
+ print(f"[received-changes] journal_read={time.monotonic() - started:.3f}s", flush=True)
703
+
704
+ def input_digest(self, spec, deadline):
705
+ tables = spec.get("observation_tables", sorted(self.paths))
706
+ parts = []
707
+ for table in tables:
708
+ files = self.paths.get(table)
709
+ if not files:
710
+ return None
711
+ for path in files:
712
+ name = str(path)
713
+ if name not in self.digests:
714
+ content = hashlib.sha256()
715
+ with open(path, "rb") as handle:
716
+ while chunk := handle.read(BASELINE_CHUNK_BYTES):
717
+ if time.monotonic() >= deadline[0]:
718
+ raise BaselineUnavailable("The observation input deadline passed.")
719
+ content.update(chunk)
720
+ self.digests[name] = content.hexdigest()
721
+ parts.append([table, self.digests[name]])
722
+ return _digest(_json_bytes(parts))
723
+
724
+
725
+ def _publication_order(rt, connector, specs, paths, document=None):
726
  """Least recently observed databases first, then the smallest, so a deadline defers in rotation."""
727
+ document = _document(rt, rt.get(STORE_KEY, fresh=True)) if document is None else document
728
+ observed = document.get("observed") or {}
729
 
730
  def priority(item):
731
  database, spec = item
 
735
  return sorted(specs.items(), key=priority)
736
 
737
 
738
+ def _project_database(rt, connector, database, spec, con, root, manifest, deadline, publication=None):
739
  """Return None when the database committed, else the named reason it was deferred."""
740
  import duckdb
741
  from core import request_timing
742
+ started = time.monotonic()
743
  try:
744
  with _query_watchdog(con, deadline):
745
+ if publication:
746
+ spec = {**spec, "input_digest": publication.input_digest(spec, deadline)}
747
+ print(f"[received-changes] {database}: input_hash={time.monotonic() - started:.3f}s", flush=True)
748
+ _snapshot(rt, connector, database, spec, con, root, manifest, deadline, publication)
749
  except request_timing.RequestDeadlineExceeded as exc:
750
  # The watchdog raises on exit after a clean body too; no cause means the database committed.
751
  return None if exc.__cause__ is None else "journal deadline passed"
 
754
  except duckdb.Error as exc:
755
  # One database's failed query defers it alone; its stored baseline stays for the next pass.
756
  return type(exc).__name__
757
+ finally:
758
+ print(f"[received-changes] {database}: total={time.monotonic() - started:.3f}s", flush=True)
759
  return None
760
 
761
 
 
786
  resolution_seconds = time.monotonic() - started
787
  snapshots_started = time.monotonic()
788
  completed = deferred = skipped = 0
789
+ publication = _ObservationPublication(rt, paths)
790
+ for database, spec in _publication_order(rt, connector, specs, paths, publication.document):
791
  # A skipped database keeps its stored baseline and leads the next publication.
792
  if time.monotonic() >= deadline[0]:
793
  skipped += 1
794
  print(f"[received-changes] {database}: skipped, journal deadline passed", flush=True)
795
  continue
796
+ reason = _project_database(rt, connector, str(database), spec, con, root, manifest, deadline, publication)
797
  if reason is None:
798
  completed += 1
799
  continue
platform/modules/competitor_watch.py CHANGED
@@ -231,12 +231,11 @@ class _TermIndex(_DocumentIndex):
231
  self._put_meta(con, meta)
232
 
233
  def _insert(self, con, rows):
234
- if not rows:
235
- return
236
  statement = self.term_sql("INSERT INTO {terms} (tenant, term, label, active, starred) "
237
- "VALUES (%s, %s, %s, %s, %s) ON CONFLICT (tenant, term) DO NOTHING")
238
- with con.cursor() as cur:
239
- cur.executemany(statement, rows)
 
240
 
241
  def seed(self, found, legacy, *, ours=None, tracking_error=None):
242
  with self._locked(prepare=True) as (con, meta):
@@ -265,13 +264,9 @@ class _TermIndex(_DocumentIndex):
265
  self._insert(con, [(self.slug, t, display_term(t), True, False) for t in found if t])
266
  # Only a checked ownership result may replace membership, including a valid empty set.
267
  if ours is not None:
268
- con.execute(self.term_sql("UPDATE {terms} SET ours = FALSE WHERE tenant = %s AND ours = TRUE"),
269
- [self.slug])
270
  self._insert(con, [(self.slug, t, display_term(t), True, False) for t in ours])
271
- if ours:
272
- with con.cursor() as cur:
273
- cur.executemany(self.term_sql("UPDATE {terms} SET ours = TRUE WHERE tenant = %s AND term = %s"),
274
- [(self.slug, t) for t in ours])
275
  if tracking_error:
276
  meta.update(trackingReady=False, trackingCause=tracking_error)
277
  elif ours is not None:
@@ -532,17 +527,21 @@ def _top3(cur, terms=None):
532
  return {}
533
  cs = 'TRY_CAST("click_share" AS DOUBLE)' if "click_share" in cols else "NULL"
534
  cv = 'TRY_CAST("conversion_share" AS DOUBLE)' if "conversion_share" in cols else "NULL"
535
- pe = 'CAST("_period_end" AS VARCHAR)' if "_period_end" in cols else "''"
536
  wanted = None if terms is None else sorted({_norm(t) for t in terms})
537
  if wanted == []:
538
  return {}
539
- where = (f' AND {_term_sql("search_term")} IN (' + ','.join('?' for _ in wanted) + ')'
540
  if wanted is not None else '')
541
  rows = cur.execute(
542
- f'SELECT "search_term", CAST("_period_start" AS VARCHAR), {pe}, "clicked_asin", '
543
  f'TRY_CAST("click_share_rank" AS INTEGER), {cs}, {cv} FROM {BA_TABLE} '
544
  f'WHERE TRY_CAST("click_share_rank" AS INTEGER) BETWEEN 1 AND 3 '
545
- f'AND ({_weekly_period_sql()}){where}', wanted or []).fetchall()
 
 
 
 
546
  out = {}
547
  for term, start, end, asin, rank, share, conv in rows:
548
  key = _norm(term)
@@ -553,10 +552,52 @@ def _top3(cur, terms=None):
553
  return out
554
 
555
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
556
  def _periods(cur):
557
  if not {"_period_start", "_period_end"} <= _ba_columns(cur):
558
  return []
559
- rows = cur.execute(f'SELECT DISTINCT CAST("_period_start" AS VARCHAR) FROM {BA_TABLE} '
560
  f'WHERE {_weekly_period_sql()}').fetchall()
561
  return sorted((str(r[0]) for r in rows if r[0] is not None), reverse=True)
562
 
@@ -600,9 +641,10 @@ def tracked_count(tenant_key, *, st=None):
600
  def _brand_source_sql(cur):
601
  """Share one dated BA/listings join between membership and bulk click-share readers."""
602
  need = {"search_term", "clicked_asin", "click_share_rank", "_period_start", "_period_end"}
603
- if cur is None or not need <= _ba_columns(cur) or "asin1" not in _columns(cur, LISTINGS_TABLE):
 
604
  raise SourceNotReady("Tracked terms need dated Brand Analytics and listing ASINs.")
605
- if not cur.execute(f"SELECT EXISTS(SELECT 1 FROM {BA_TABLE} WHERE {_weekly_period_sql()})").fetchone()[0]:
606
  raise SourceNotReady(BA_WEEKLY_MISSING)
607
  return (f"WITH weekly AS (SELECT * FROM {BA_TABLE} WHERE {_weekly_period_sql()}), "
608
  f"own_asins AS (SELECT DISTINCT trim(asin1) AS asin FROM {LISTINGS_TABLE} "
@@ -639,8 +681,23 @@ def brand_share_source(cur, terms, start=None, end=None):
639
  yield from batch
640
 
641
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
642
  def seed_terms(tenant_key, *, st=None, cur=None):
643
- """Add every distinct SQP search query to the seeded set. A removed term stays removed."""
644
  st = _rt(tenant_key, st)
645
  index = _term_index(tenant_key, st)
646
 
@@ -650,22 +707,13 @@ def seed_terms(tenant_key, *, st=None, cur=None):
650
  cause = str(exc) if isinstance(exc, _ReadFailed) else type(exc).__name__
651
  return f"Tracked terms could not be checked ({cause}). The next Amazon sync will retry."
652
 
653
- def distinct(reader):
654
- if reader is None or "search_query_data_search_query" not in _columns(reader, SQP_TABLE):
655
- if reader is not None and _columns(reader, BA_TABLE):
656
- return set()
657
- raise TermsNotReady(TERMS_NOT_READY)
658
- return {r[0] for r in reader.execute(f'SELECT DISTINCT {_term_sql("search_query_data_search_query")} '
659
- f'AS term FROM {SQP_TABLE} WHERE {_term_sql("search_query_data_search_query")} <> \'\'').fetchall()}
660
-
661
  def source(reader):
662
  try:
663
- found = distinct(reader)
664
- except Exception: # noqa: BLE001
665
  if index is None:
666
  raise
667
- # Discovery only adds retention rows, so a failed read must keep its existing list.
668
- found = set()
669
  try:
670
  ours = _tracked_terms_source(reader)
671
  except Exception as exc: # noqa: BLE001
@@ -1050,6 +1098,21 @@ def _clear_error(doc):
1050
  return doc
1051
 
1052
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1053
  def compare(tenant_key, synced_tables=("brand_analytics",), *, st=None, now=None, quote_asins=None):
1054
  """Compare the newest held week with the week before and raise owned alerts. Returns a summary."""
1055
  st = _rt(tenant_key, st)
@@ -1073,104 +1136,40 @@ def compare(tenant_key, synced_tables=("brand_analytics",), *, st=None, now=None
1073
  terms = _comparison_terms(tenant_key, st)
1074
  held = _top3(cur, terms)
1075
  ours = _listing_asins(cur)
1076
- weeks = _doc(st, KEY_WEEKS)
1077
  newest = periods[0]
1078
  if daily and weeks.get("last_compared") != newest:
1079
  return {"state": "awaits_weekly_pull", "period": newest}
1080
  retry = set(weeks.get("pending_quotes") or ()) & set(quote_asins or ()) if daily else None
1081
  if daily and not retry:
1082
  return {"state": "already_compared", "period": newest}
1083
- if not daily and weeks.get("last_compared") == newest:
1084
- # Every read this pass succeeded on a compared week, so a stored sync failure is superseded.
1085
- if "last_error" in weeks:
1086
- st.update(KEY_WEEKS, _clear_error)
1087
  return {"state": "already_compared", "period": newest}
1088
  previous = periods[1] if len(periods) > 1 else None
1089
  conf = settings(tenant_key, st=st)
1090
 
1091
- our_for = {t: _comparison_asin((held.get(t) or {}).get(newest), ours) for t in terms}
1092
- want = set()
1093
- for term in terms:
1094
- want |= set(((held.get(term) or {}).get(newest) or {}).get("rows", {}))
1095
- if our_for.get(term):
1096
- want.add(our_for[term])
1097
  quotes, quote_cause = _daily_quotes(tenant_key, st, want)
1098
  current = (weeks.get("metrics") or {}).get(newest) or {}
1099
  metrics = {a: current[a] if daily and a not in retry and a in current else
1100
  _quote_metric(quotes.get(a), a in ours, quote_cause) for a in sorted(want)}
1101
  prior = ((weeks.get("metrics") or {}).get(previous) or {}) if previous else {}
1102
 
1103
- raised, checks = [], []
 
1104
  for asin, metric in metrics.items():
1105
  parts = list(dict.fromkeys(c for c in (metric.get("priceCause"), metric.get("rankCause")) if c))
1106
  if metric.get("price") is not None:
1107
  parts.insert(0, f"Price {metric['price']:,.2f} {metric.get('currency') or ''}.".strip())
1108
  checks.append({"term": "", "asin": asin, "trigger": "Daily price", "periodStart": newest,
1109
  "detail": metric, "cause": " ".join(parts)})
1110
- if previous is not None:
1111
- undercut = conf["undercut_pct"] / 100.0
1112
- surge = conf["rank_surge_pct"] / 100.0
1113
- for term in terms:
1114
- now_week = (held.get(term) or {}).get(newest)
1115
- was_week = (held.get(term) or {}).get(previous)
1116
- if not now_week or not was_week:
1117
- continue
1118
- top_now, top_was = now_week["rows"], was_week["rows"]
1119
- end = now_week["end"]
1120
-
1121
- def found(asin, key, detail, cause=None):
1122
- raised.append({"term": term, "asin": asin, "trigger": key, "periodStart": newest,
1123
- "periodEnd": end, "previousPeriodStart": previous, "detail": detail,
1124
- "cause": cause})
1125
-
1126
- def asin_cause(asin):
1127
- m = metrics.get(asin) or {}
1128
- return " ".join(c for c in (m.get("priceCause"), m.get("rankCause")) if c) or None
1129
-
1130
- for asin in sorted(set(top_now) - set(top_was)) if not daily else ():
1131
- found(asin, "new_asin", {"rank": top_now[asin][0]}, asin_cause(asin))
1132
- for asin in sorted((set(top_was) - set(top_now)) & ours) if not daily else ():
1133
- found(asin, "lost_spot", {"previousRank": top_was[asin][0]})
1134
- mine = our_for.get(term)
1135
- mine_m = metrics.get(mine) or {}
1136
- if mine is None and set(top_now) - ours:
1137
- checks.append({"term": term, "asin": None, "trigger": TRIGGERS["undercut"],
1138
- "periodStart": newest,
1139
- "cause": "None of our listing ASINs is in this term's top 3, so there is "
1140
- "no price of ours to compare."})
1141
- for asin in sorted(set(top_now) - ours):
1142
- m = metrics.get(asin) or {}
1143
- if mine is None:
1144
- pass
1145
- elif mine_m.get("price") is None or m.get("price") is None:
1146
- cause = " ".join(c for c in (mine_m.get("priceCause") and f"Our price: {mine_m['priceCause']}",
1147
- m.get("priceCause")) if c)
1148
- checks.append({"term": term, "asin": asin, "trigger": TRIGGERS["undercut"],
1149
- "periodStart": newest, "cause": cause or "No price was read."})
1150
- elif m["price"] <= mine_m["price"] * (1 - undercut) and (not daily or asin in retry or mine in retry):
1151
- found(asin, "undercut", {"ourAsin": mine, "ourPrice": mine_m["price"],
1152
- "price": m["price"],
1153
- "percentUnder": round(100 * (1 - m["price"] / mine_m["price"]), 2)},
1154
- asin_cause(asin))
1155
- old = (prior.get(asin) or {})
1156
- if m.get("rank") is None:
1157
- checks.append({"term": term, "asin": asin, "trigger": TRIGGERS["rank_surge"],
1158
- "periodStart": newest, "cause": m.get("rankCause") or "No sales rank was read."})
1159
- elif old.get("rank") is None:
1160
- if old.get("rankCause"):
1161
- checks.append({"term": term, "asin": asin, "trigger": TRIGGERS["rank_surge"],
1162
- "periodStart": newest,
1163
- "cause": f"The week before: {old['rankCause']}"})
1164
- elif m["rank"] <= old["rank"] * (1 - surge) and (not daily or asin in retry):
1165
- found(asin, "rank_surge", {"rank": m["rank"], "previousRank": old["rank"]},
1166
- asin_cause(asin))
1167
-
1168
  created = _alert_index(tenant_key, st).record(raised, checks, newest, conf["defaultOwner"], now)
1169
 
1170
  ready = previous is not None
1171
 
1172
  def _weeks(doc):
1173
- doc = doc if isinstance(doc, dict) else {}
1174
  doc.setdefault("metrics", {})[newest] = metrics
1175
  doc["pending_quotes"] = [asin for asin, metric in metrics.items()
1176
  if metric.get("price") is None or metric.get("rank") is None]
@@ -1185,6 +1184,64 @@ def compare(tenant_key, synced_tables=("brand_analytics",), *, st=None, now=None
1185
  "created": created, "checks": checks}
1186
 
1187
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1188
  # --------------------------------------------------------------------------- alerts
1189
 
1190
  class _AlertIndex(_DocumentIndex):
@@ -1395,20 +1452,22 @@ def update_alert(tenant_key, alert_id, *, owner=None, status=None, due=None, use
1395
 
1396
  # --------------------------------------------------------------------------- term detail
1397
 
1398
- def term_detail(tenant_key, term, *, st=None):
1399
  """One term's held periods, the newest top 3 with week-over-week change, and the click share trend."""
1400
  st = _rt(tenant_key, st)
1401
  key = _norm(term)
1402
- with _reader(tenant_key, st) as cur:
1403
  sqp = _sqp(cur, key) if cur is not None else []
1404
- weeks_held = _top3(cur, [key]).get(key, {})
1405
  candidates = {a for week in weeks_held.values() for a in week["rows"]}
1406
  ours = _our_asins(cur, sqp, candidates) if cur is not None else set()
1407
  periods = sorted(weeks_held, reverse=True)
1408
  metric_asins = sorted(weeks_held[periods[0]]["rows"]) if periods else []
 
1409
  state, *values = _paths(tenant_key, st, KEY_WEEKS, [("comparison",)] +
1410
- [("metrics", periods[0], asin) for asin in metric_asins])
1411
- metric = dict(zip(metric_asins, values))
 
1412
  top3 = []
1413
  if periods:
1414
  newest = weeks_held[periods[0]]["rows"]
 
231
  self._put_meta(con, meta)
232
 
233
  def _insert(self, con, rows):
 
 
234
  statement = self.term_sql("INSERT INTO {terms} (tenant, term, label, active, starred) "
235
+ "SELECT * FROM unnest(%s::text[], %s::text[], %s::text[], %s::boolean[], %s::boolean[]) "
236
+ "ON CONFLICT (tenant, term) DO NOTHING")
237
+ for start in range(0, len(rows), 1000):
238
+ con.execute(statement, [list(values) for values in zip(*rows[start:start + 1000])])
239
 
240
  def seed(self, found, legacy, *, ours=None, tracking_error=None):
241
  with self._locked(prepare=True) as (con, meta):
 
264
  self._insert(con, [(self.slug, t, display_term(t), True, False) for t in found if t])
265
  # Only a checked ownership result may replace membership, including a valid empty set.
266
  if ours is not None:
 
 
267
  self._insert(con, [(self.slug, t, display_term(t), True, False) for t in ours])
268
+ con.execute(self.term_sql("UPDATE {terms} SET ours = (term = ANY(%s)) WHERE tenant = %s "
269
+ "AND ours IS DISTINCT FROM (term = ANY(%s))"), [list(ours), self.slug, list(ours)])
 
 
270
  if tracking_error:
271
  meta.update(trackingReady=False, trackingCause=tracking_error)
272
  elif ours is not None:
 
527
  return {}
528
  cs = 'TRY_CAST("click_share" AS DOUBLE)' if "click_share" in cols else "NULL"
529
  cv = 'TRY_CAST("conversion_share" AS DOUBLE)' if "conversion_share" in cols else "NULL"
530
+ pe = 'CAST(TRY_CAST("_period_end" AS DATE) AS VARCHAR)'
531
  wanted = None if terms is None else sorted({_norm(t) for t in terms})
532
  if wanted == []:
533
  return {}
534
+ where = (f' AND {_term_sql("search_term")} IN (SELECT unnest(?::VARCHAR[]))'
535
  if wanted is not None else '')
536
  rows = cur.execute(
537
+ f'SELECT "search_term", CAST(TRY_CAST("_period_start" AS DATE) AS VARCHAR), {pe}, "clicked_asin", '
538
  f'TRY_CAST("click_share_rank" AS INTEGER), {cs}, {cv} FROM {BA_TABLE} '
539
  f'WHERE TRY_CAST("click_share_rank" AS INTEGER) BETWEEN 1 AND 3 '
540
+ f'AND ({_weekly_period_sql()}){where}', [wanted] if wanted is not None else []).fetchall()
541
+ return _ranked_history(rows, wanted)
542
+
543
+
544
+ def _ranked_history(rows, wanted=None):
545
  out = {}
546
  for term, start, end, asin, rank, share, conv in rows:
547
  key = _norm(term)
 
552
  return out
553
 
554
 
555
+ def _page_history(cur, terms, selected):
556
+ """Read page week counts and the selected term's ranks in one source scan."""
557
+ cols = _ba_columns(cur)
558
+ wanted = sorted(set(terms) | ({selected} if selected else set()))
559
+ if not wanted or not {"search_term", "_period_start", "_period_end"} <= cols:
560
+ return {}, {}
561
+ optional = {name: f'TRY_CAST("{name}" AS {kind})' if name in cols else f'NULL::{kind}'
562
+ for name, kind in (("clicked_asin", "VARCHAR"), ("click_share_rank", "INTEGER"),
563
+ ("click_share", "DOUBLE"), ("conversion_share", "DOUBLE"))}
564
+ query = (f'WITH held AS MATERIALIZED (SELECT {_term_sql("search_term")} AS term, '
565
+ 'CAST(TRY_CAST("_period_start" AS DATE) AS VARCHAR) AS start, '
566
+ 'CAST(TRY_CAST("_period_end" AS DATE) AS VARCHAR) AS stop, '
567
+ + ', '.join(f'{expr} AS "{name}"' for name, expr in optional.items()) +
568
+ f' FROM {BA_TABLE} WHERE {_weekly_period_sql()} '
569
+ f'AND {_term_sql("search_term")} IN (SELECT unnest(?::VARCHAR[]))) '
570
+ 'SELECT term, COUNT(DISTINCT start) AS weeks, NULL AS start, NULL AS stop, '
571
+ 'NULL AS asin, NULL::INTEGER AS rank, NULL::DOUBLE AS share, NULL::DOUBLE AS conversion '
572
+ 'FROM held GROUP BY term UNION ALL '
573
+ 'SELECT term, NULL, start, stop, clicked_asin, click_share_rank, click_share, conversion_share '
574
+ 'FROM held WHERE term = ? AND click_share_rank BETWEEN 1 AND 3')
575
+ rows = cur.execute(query, [wanted, selected]).fetchall()
576
+ counts = {term: weeks for term, weeks, *_rest in rows if weeks is not None}
577
+ history = _ranked_history([(term, *values) for term, weeks, *values in rows if weeks is None])
578
+ return counts, history.get(selected, {})
579
+
580
+
581
+ def term_page_history(tenant_key, page, *, st=None):
582
+ """Share the page's weekly scan with selected-term detail without preparing stored state."""
583
+ st = _rt(tenant_key, st)
584
+ selected = page["term"]
585
+ if not page["terms"] and not selected:
586
+ return {}, {}
587
+ try:
588
+ with _reader(tenant_key, st) as cur:
589
+ counts, history = _page_history(cur, page["terms"], selected)
590
+ detail = term_detail(tenant_key, selected, st=st, reader=cur, history=history) if selected else {}
591
+ except Exception as exc:
592
+ cause = str(exc) if isinstance(exc, _ReadFailed) else type(exc).__name__
593
+ raise SourceNotReady(cause, page=page) from exc
594
+ return counts, detail
595
+
596
+
597
  def _periods(cur):
598
  if not {"_period_start", "_period_end"} <= _ba_columns(cur):
599
  return []
600
+ rows = cur.execute(f'SELECT DISTINCT CAST(TRY_CAST("_period_start" AS DATE) AS VARCHAR) FROM {BA_TABLE} '
601
  f'WHERE {_weekly_period_sql()}').fetchall()
602
  return sorted((str(r[0]) for r in rows if r[0] is not None), reverse=True)
603
 
 
641
  def _brand_source_sql(cur):
642
  """Share one dated BA/listings join between membership and bulk click-share readers."""
643
  need = {"search_term", "clicked_asin", "click_share_rank", "_period_start", "_period_end"}
644
+ cols, weekly = _ba_source(cur)
645
+ if cur is None or not need <= cols or "asin1" not in _columns(cur, LISTINGS_TABLE):
646
  raise SourceNotReady("Tracked terms need dated Brand Analytics and listing ASINs.")
647
+ if not weekly:
648
  raise SourceNotReady(BA_WEEKLY_MISSING)
649
  return (f"WITH weekly AS (SELECT * FROM {BA_TABLE} WHERE {_weekly_period_sql()}), "
650
  f"own_asins AS (SELECT DISTINCT trim(asin1) AS asin FROM {LISTINGS_TABLE} "
 
681
  yield from batch
682
 
683
 
684
+ def _discover_terms(reader):
685
+ """Union the two source vocabularies without treating SQP coverage as BA membership."""
686
+ if reader is None:
687
+ raise TermsNotReady(TERMS_NOT_READY)
688
+ sources = []
689
+ if "search_query_data_search_query" in _columns(reader, SQP_TABLE):
690
+ sources.append(f'SELECT {_term_sql("search_query_data_search_query")} AS term FROM {SQP_TABLE}')
691
+ if {"search_term", "_period_start", "_period_end"} <= _columns(reader, BA_TABLE):
692
+ sources.append(f'SELECT {_term_sql("search_term")} AS term FROM {BA_TABLE} WHERE {_weekly_period_sql()}')
693
+ if not sources:
694
+ raise TermsNotReady(TERMS_NOT_READY)
695
+ return {row[0] for row in reader.execute(
696
+ 'SELECT DISTINCT term FROM (' + ' UNION ALL '.join(sources) + ") WHERE term <> ''").fetchall()}
697
+
698
+
699
  def seed_terms(tenant_key, *, st=None, cur=None):
700
+ """Discover SQP queries and held weekly BA terms while preserving authored membership."""
701
  st = _rt(tenant_key, st)
702
  index = _term_index(tenant_key, st)
703
 
 
707
  cause = str(exc) if isinstance(exc, _ReadFailed) else type(exc).__name__
708
  return f"Tracked terms could not be checked ({cause}). The next Amazon sync will retry."
709
 
 
 
 
 
 
 
 
 
710
  def source(reader):
711
  try:
712
+ found = _discover_terms(reader)
713
+ except Exception as exc:
714
  if index is None:
715
  raise
716
+ return set(), None, failure_cause(exc)
 
717
  try:
718
  ours = _tracked_terms_source(reader)
719
  except Exception as exc: # noqa: BLE001
 
1098
  return doc
1099
 
1100
 
1101
+ def _weekly_state(doc):
1102
+ """Normalize legacy timestamp week keys before comparing with connector date keys."""
1103
+ out = dict(doc)
1104
+ if out.get("last_compared"):
1105
+ out["last_compared"] = dt.date.fromisoformat(str(out["last_compared"])[:10]).isoformat()
1106
+ metrics = {}
1107
+ for key, value in (out.get("metrics") or {}).items():
1108
+ day = dt.date.fromisoformat(str(key)[:10]).isoformat()
1109
+ if day in metrics and metrics[day] != value:
1110
+ raise SourceNotReady("Stored weekly metrics disagree for one week.")
1111
+ metrics[day] = value
1112
+ out["metrics"] = metrics
1113
+ return out
1114
+
1115
+
1116
  def compare(tenant_key, synced_tables=("brand_analytics",), *, st=None, now=None, quote_asins=None):
1117
  """Compare the newest held week with the week before and raise owned alerts. Returns a summary."""
1118
  st = _rt(tenant_key, st)
 
1136
  terms = _comparison_terms(tenant_key, st)
1137
  held = _top3(cur, terms)
1138
  ours = _listing_asins(cur)
1139
+ weeks = _weekly_state(_doc(st, KEY_WEEKS))
1140
  newest = periods[0]
1141
  if daily and weeks.get("last_compared") != newest:
1142
  return {"state": "awaits_weekly_pull", "period": newest}
1143
  retry = set(weeks.get("pending_quotes") or ()) & set(quote_asins or ()) if daily else None
1144
  if daily and not retry:
1145
  return {"state": "already_compared", "period": newest}
1146
+ if not daily and weeks.get("last_compared") == newest and not weeks.get("last_error"):
 
 
 
1147
  return {"state": "already_compared", "period": newest}
1148
  previous = periods[1] if len(periods) > 1 else None
1149
  conf = settings(tenant_key, st=st)
1150
 
1151
+ want = {asin for term in terms for asin in ((held.get(term) or {}).get(newest) or {}).get("rows", {})}
 
 
 
 
 
1152
  quotes, quote_cause = _daily_quotes(tenant_key, st, want)
1153
  current = (weeks.get("metrics") or {}).get(newest) or {}
1154
  metrics = {a: current[a] if daily and a not in retry and a in current else
1155
  _quote_metric(quotes.get(a), a in ours, quote_cause) for a in sorted(want)}
1156
  prior = ((weeks.get("metrics") or {}).get(previous) or {}) if previous else {}
1157
 
1158
+ raised, term_checks = _term_signals(terms, held, periods, ours, metrics, prior, conf, retry)
1159
+ checks = []
1160
  for asin, metric in metrics.items():
1161
  parts = list(dict.fromkeys(c for c in (metric.get("priceCause"), metric.get("rankCause")) if c))
1162
  if metric.get("price") is not None:
1163
  parts.insert(0, f"Price {metric['price']:,.2f} {metric.get('currency') or ''}.".strip())
1164
  checks.append({"term": "", "asin": asin, "trigger": "Daily price", "periodStart": newest,
1165
  "detail": metric, "cause": " ".join(parts)})
1166
+ checks.extend(term_checks)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1167
  created = _alert_index(tenant_key, st).record(raised, checks, newest, conf["defaultOwner"], now)
1168
 
1169
  ready = previous is not None
1170
 
1171
  def _weeks(doc):
1172
+ doc = _weekly_state(doc if isinstance(doc, dict) else {})
1173
  doc.setdefault("metrics", {})[newest] = metrics
1174
  doc["pending_quotes"] = [asin for asin, metric in metrics.items()
1175
  if metric.get("price") is None or metric.get("rank") is None]
 
1184
  "created": created, "checks": checks}
1185
 
1186
 
1187
+ def _term_signals(terms, held, periods, ours, metrics, prior, conf, retry):
1188
+ """Derive weekly and quote-retry signals without changing alert state."""
1189
+ raised, checks = [], []
1190
+ if len(periods) < 2:
1191
+ return raised, checks
1192
+ newest, previous = periods[:2]
1193
+ daily = retry is not None
1194
+ undercut, surge = conf["undercut_pct"] / 100.0, conf["rank_surge_pct"] / 100.0
1195
+ for term in terms:
1196
+ now_week = (held.get(term) or {}).get(newest)
1197
+ was_week = (held.get(term) or {}).get(previous)
1198
+ if not now_week or not was_week:
1199
+ continue
1200
+ top_now, top_was = now_week["rows"], was_week["rows"]
1201
+
1202
+ def found(asin, key, detail, cause=None):
1203
+ raised.append({"term": term, "asin": asin, "trigger": key, "periodStart": newest,
1204
+ "periodEnd": now_week["end"], "previousPeriodStart": previous,
1205
+ "detail": detail, "cause": cause})
1206
+
1207
+ def asin_cause(asin):
1208
+ m = metrics.get(asin) or {}
1209
+ return " ".join(c for c in (m.get("priceCause"), m.get("rankCause")) if c) or None
1210
+
1211
+ for asin in sorted(set(top_now) - set(top_was)) if not daily else ():
1212
+ found(asin, "new_asin", {"rank": top_now[asin][0]}, asin_cause(asin))
1213
+ for asin in sorted((set(top_was) - set(top_now)) & ours) if not daily else ():
1214
+ found(asin, "lost_spot", {"previousRank": top_was[asin][0]})
1215
+ mine = _comparison_asin(now_week, ours)
1216
+ mine_m = metrics.get(mine) or {}
1217
+ if mine is None and set(top_now) - ours:
1218
+ checks.append({"term": term, "asin": None, "trigger": TRIGGERS["undercut"], "periodStart": newest,
1219
+ "cause": "None of our listing ASINs is in this term's top 3, so there is "
1220
+ "no price of ours to compare."})
1221
+ for asin in sorted(set(top_now) - ours):
1222
+ m = metrics.get(asin) or {}
1223
+ if mine is not None:
1224
+ if mine_m.get("price") is None or m.get("price") is None:
1225
+ cause = " ".join(c for c in (mine_m.get("priceCause") and f"Our price: {mine_m['priceCause']}",
1226
+ m.get("priceCause")) if c)
1227
+ checks.append({"term": term, "asin": asin, "trigger": TRIGGERS["undercut"],
1228
+ "periodStart": newest, "cause": cause or "No price was read."})
1229
+ elif m["price"] <= mine_m["price"] * (1 - undercut) and (not daily or asin in retry or mine in retry):
1230
+ found(asin, "undercut", {"ourAsin": mine, "ourPrice": mine_m["price"], "price": m["price"],
1231
+ "percentUnder": round(100 * (1 - m["price"] / mine_m["price"]), 2)}, asin_cause(asin))
1232
+ old = prior.get(asin) or {}
1233
+ if m.get("rank") is None:
1234
+ checks.append({"term": term, "asin": asin, "trigger": TRIGGERS["rank_surge"],
1235
+ "periodStart": newest, "cause": m.get("rankCause") or "No sales rank was read."})
1236
+ elif old.get("rank") is None:
1237
+ if old.get("rankCause"):
1238
+ checks.append({"term": term, "asin": asin, "trigger": TRIGGERS["rank_surge"],
1239
+ "periodStart": newest, "cause": f"The week before: {old['rankCause']}"})
1240
+ elif m["rank"] <= old["rank"] * (1 - surge) and (not daily or asin in retry):
1241
+ found(asin, "rank_surge", {"rank": m["rank"], "previousRank": old["rank"]}, asin_cause(asin))
1242
+ return raised, checks
1243
+
1244
+
1245
  # --------------------------------------------------------------------------- alerts
1246
 
1247
  class _AlertIndex(_DocumentIndex):
 
1452
 
1453
  # --------------------------------------------------------------------------- term detail
1454
 
1455
+ def term_detail(tenant_key, term, *, st=None, reader=None, history=None):
1456
  """One term's held periods, the newest top 3 with week-over-week change, and the click share trend."""
1457
  st = _rt(tenant_key, st)
1458
  key = _norm(term)
1459
+ with contextlib.nullcontext(reader) if reader is not None else _reader(tenant_key, st) as cur:
1460
  sqp = _sqp(cur, key) if cur is not None else []
1461
+ weeks_held = _top3(cur, [key]).get(key, {}) if history is None else history
1462
  candidates = {a for week in weeks_held.values() for a in week["rows"]}
1463
  ours = _our_asins(cur, sqp, candidates) if cur is not None else set()
1464
  periods = sorted(weeks_held, reverse=True)
1465
  metric_asins = sorted(weeks_held[periods[0]]["rows"]) if periods else []
1466
+ variants = (periods[0], periods[0] + "T00:00:00Z", periods[0] + " 00:00:00") if periods else ()
1467
  state, *values = _paths(tenant_key, st, KEY_WEEKS, [("comparison",)] +
1468
+ [("metrics", week, asin) for asin in metric_asins for week in variants])
1469
+ metric = {asin: next((value for value in values[i * 3:i * 3 + 3] if value is not None), None)
1470
+ for i, asin in enumerate(metric_asins)}
1471
  top3 = []
1472
  if periods:
1473
  newest = weeks_held[periods[0]]["rows"]
web/public/AIOS_DEPLOY_RECEIPT CHANGED
@@ -1 +1 @@
1
- ee4618c92d8cbec218349feb4f5f9fbc6cb19f8ba0f0e63314aaa24270742c00
 
1
+ bb954b673746f91c1826c1869e08268a546c64e816c8669fbb9654a6ee4bb30d