Deploy AIOS web (React glide grid + FastAPI slice)
Browse files- RELEASES.json +1 -1
- VERSION +1 -1
- api/external_activity.py +33 -7
- api/main.py +11 -3
- api/registry_source.py +27 -3
- api/request_timing.py +25 -4
- api/routes_opportunity_finder.py +16 -6
- api/routes_sheet.py +182 -59
- platform/evals/royal_questions.yml +1 -1
- platform/harness/amazon_store.py +62 -12
- platform/harness/datastore.py +11 -3
- platform/harness/semantic.py +42 -4
- platform/model/metrics/receivables.yml +12 -7
- platform/model/skills/odoo/overdue-receivables.md +5 -3
- platform/model/topics/gl_lines.yml +2 -0
- platform/modules/competitor_watch.py +25 -15
- platform/modules/opportunity_finder.py +107 -62
- platform/modules/ranking_diagnostic.py +42 -22
- platform/modules/restock_planner.py +53 -28
- web/public/AIOS_DEPLOY_RECEIPT +1 -1
RELEASES.json
CHANGED
|
@@ -1,4 +1,4 @@
|
|
| 1 |
{
|
| 2 |
-
"current": "
|
| 3 |
"releases": []
|
| 4 |
}
|
|
|
|
| 1 |
{
|
| 2 |
+
"current": "23787a8ce",
|
| 3 |
"releases": []
|
| 4 |
}
|
VERSION
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
|
|
|
|
| 1 |
+
23787a8ce
|
api/external_activity.py
CHANGED
|
@@ -3,12 +3,20 @@
|
|
| 3 |
from contextlib import contextmanager
|
| 4 |
from contextvars import ContextVar
|
| 5 |
import json
|
|
|
|
|
|
|
| 6 |
from core import audit, external_access
|
| 7 |
|
| 8 |
|
| 9 |
RETAINED_MAX = audit.MAX_EVENTS
|
| 10 |
_CALL = ContextVar("external_activity_call", default=None)
|
| 11 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 12 |
|
| 13 |
def _preview(session):
|
| 14 |
return bool(getattr(session.runtime, "preview_only", False)
|
|
@@ -179,13 +187,7 @@ def _text(value):
|
|
| 179 |
def _detail(session, database, outcome, code):
|
| 180 |
claims = session.claims or {}
|
| 181 |
key_id = _text(claims.get("external_key_id") or claims.get("external_credential_id"))
|
| 182 |
-
label = ""
|
| 183 |
-
if key_id:
|
| 184 |
-
try:
|
| 185 |
-
rows = external_access.list_for_user(session.runtime, session.uname, session.tenant)
|
| 186 |
-
label = next((row.get("label") for row in rows if row.get("id") == key_id), "")
|
| 187 |
-
except external_access.ExternalAccessError:
|
| 188 |
-
pass
|
| 189 |
return {
|
| 190 |
"key_id": key_id, "key_label": _text(label), "database": _text(database),
|
| 191 |
"outcome": outcome, "code": _text(code),
|
|
@@ -193,6 +195,30 @@ def _detail(session, database, outcome, code):
|
|
| 193 |
}
|
| 194 |
|
| 195 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 196 |
def _record(session, database, outcome, code, *, tool="query_database", action="query", via=None, call=None,
|
| 197 |
method=None):
|
| 198 |
if _preview(session) or _CALL.get() is not None:
|
|
|
|
| 3 |
from contextlib import contextmanager
|
| 4 |
from contextvars import ContextVar
|
| 5 |
import json
|
| 6 |
+
import threading
|
| 7 |
+
import time
|
| 8 |
from core import audit, external_access
|
| 9 |
|
| 10 |
|
| 11 |
RETAINED_MAX = audit.MAX_EVENTS
|
| 12 |
_CALL = ContextVar("external_activity_call", default=None)
|
| 13 |
|
| 14 |
+
#: (tenant, user, key id) -> (label, monotonic stamp). A rename shows in the audit within the TTL.
|
| 15 |
+
_LABELS = {}
|
| 16 |
+
_LABELS_LOCK = threading.Lock()
|
| 17 |
+
LABEL_TTL_SECONDS = 60.0
|
| 18 |
+
LABEL_CACHE_MAX = 512
|
| 19 |
+
|
| 20 |
|
| 21 |
def _preview(session):
|
| 22 |
return bool(getattr(session.runtime, "preview_only", False)
|
|
|
|
| 187 |
def _detail(session, database, outcome, code):
|
| 188 |
claims = session.claims or {}
|
| 189 |
key_id = _text(claims.get("external_key_id") or claims.get("external_credential_id"))
|
| 190 |
+
label = _key_label(session, key_id) if key_id else ""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 191 |
return {
|
| 192 |
"key_id": key_id, "key_label": _text(label), "database": _text(database),
|
| 193 |
"outcome": outcome, "code": _text(code),
|
|
|
|
| 195 |
}
|
| 196 |
|
| 197 |
|
| 198 |
+
def _key_label(session, key_id):
|
| 199 |
+
"""The key's label, read from the store at most once per key per TTL in this process."""
|
| 200 |
+
owner = (str(session.tenant or "").lower(), str(session.uname or "").lower())
|
| 201 |
+
now = time.monotonic()
|
| 202 |
+
with _LABELS_LOCK:
|
| 203 |
+
hit = _LABELS.get((*owner, key_id))
|
| 204 |
+
if hit is not None and now - hit[1] < LABEL_TTL_SECONDS:
|
| 205 |
+
return hit[0]
|
| 206 |
+
try:
|
| 207 |
+
rows = external_access.list_for_user(session.runtime, session.uname, session.tenant)
|
| 208 |
+
except external_access.ExternalAccessError:
|
| 209 |
+
return ""
|
| 210 |
+
found = {str(row.get("id")): str(row.get("label") or "") for row in rows if row.get("id")}
|
| 211 |
+
with _LABELS_LOCK:
|
| 212 |
+
if len(_LABELS) + len(found) + 1 > LABEL_CACHE_MAX:
|
| 213 |
+
_LABELS.clear()
|
| 214 |
+
for row_id, row_label in found.items():
|
| 215 |
+
_LABELS[(*owner, row_id)] = (row_label, now)
|
| 216 |
+
# A key with no row (an OAuth credential, a revoked key) caches its miss for the TTL too.
|
| 217 |
+
if key_id not in found:
|
| 218 |
+
_LABELS[(*owner, key_id)] = ("", now)
|
| 219 |
+
return found.get(key_id, "")
|
| 220 |
+
|
| 221 |
+
|
| 222 |
def _record(session, database, outcome, code, *, tool="query_database", action="query", via=None, call=None,
|
| 223 |
method=None):
|
| 224 |
if _preview(session) or _CALL.get() is not None:
|
api/main.py
CHANGED
|
@@ -1328,8 +1328,13 @@ def _run_sync_requests(outcome=None, *, during_odoo=False):
|
|
| 1328 |
odoo._tlocal.tenant_ctx = None
|
| 1329 |
from connector_onboarding import selected_mirror_entities
|
| 1330 |
selected_entities = selected_mirror_entities(rt)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1331 |
report = mirror_store.tracked_sync("odoo", rt, datastore.sync_all, entities=selected_entities,
|
| 1332 |
-
log=lambda *a, **k: None
|
|
|
|
| 1333 |
finally:
|
| 1334 |
odoo._tlocal.client = previous_client
|
| 1335 |
odoo._tlocal.tenant_ctx = previous_context
|
|
@@ -2526,8 +2531,10 @@ def _capture_resync_source_counts(tenant):
|
|
| 2526 |
rest = _rel_counts.enabled_buckets(rt) - set(during)
|
| 2527 |
if rest:
|
| 2528 |
counts.update(_rel_counts.capture_source_counts(rt, buckets=rest))
|
|
|
|
| 2529 |
print(f"[aios-api] store resync: counted the Odoo source for {len(counts)} "
|
| 2530 |
-
f"database(s) at mirror-pull time, {len(during)} beside their own copy"
|
|
|
|
| 2531 |
return counts or None
|
| 2532 |
except Exception as e: # noqa: BLE001
|
| 2533 |
print(f"[aios-api] store resync: the Odoo source was not counted at mirror-pull time "
|
|
@@ -2803,7 +2810,8 @@ def _store_resync_loop():
|
|
| 2803 |
# D-9131: count the source now, before the vendor pulls, so reconciliation compares the
|
| 2804 |
# mirror with the source as of the same moment. A pass without a good pull counts nothing.
|
| 2805 |
_source_counts = (_capture_resync_source_counts(_cad["tenant"])
|
| 2806 |
-
if _err is None and sync_report and not _manual
|
|
|
|
| 2807 |
# ⭐ WAVE 27 item 17 (E's ASK ->A, resolved): whenever the Odoo RELATIONAL tables need
|
| 2808 |
# refreshing, they run AFTER the mirror they derive from. A confirmed zero-change report
|
| 2809 |
# is the W39 exception; deriving before a changed mirror advances would publish a one-cycle
|
|
|
|
| 1328 |
odoo._tlocal.tenant_ctx = None
|
| 1329 |
from connector_onboarding import selected_mirror_entities
|
| 1330 |
selected_entities = selected_mirror_entities(rt)
|
| 1331 |
+
# Sync now counts each entity's source beside its copy, as the full pass does.
|
| 1332 |
+
# Looked up by name: extracted test fixtures of this function omit the counter.
|
| 1333 |
+
_counter = globals().get("_entity_source_counter")
|
| 1334 |
+
_after_entity = _counter(datastore) if _counter else None
|
| 1335 |
report = mirror_store.tracked_sync("odoo", rt, datastore.sync_all, entities=selected_entities,
|
| 1336 |
+
log=lambda *a, **k: None,
|
| 1337 |
+
**({"after_entity": _after_entity} if _after_entity else {}))
|
| 1338 |
finally:
|
| 1339 |
odoo._tlocal.client = previous_client
|
| 1340 |
odoo._tlocal.tenant_ctx = previous_context
|
|
|
|
| 2531 |
rest = _rel_counts.enabled_buckets(rt) - set(during)
|
| 2532 |
if rest:
|
| 2533 |
counts.update(_rel_counts.capture_source_counts(rt, buckets=rest))
|
| 2534 |
+
per_entity = " ".join(f"{bucket}={count:,}" for bucket, (count, _mode) in sorted(during.items()))
|
| 2535 |
print(f"[aios-api] store resync: counted the Odoo source for {len(counts)} "
|
| 2536 |
+
f"database(s) at mirror-pull time, {len(during)} beside their own copy"
|
| 2537 |
+
+ (f"; source counts per entity: {per_entity}" if per_entity else ""), flush=True)
|
| 2538 |
return counts or None
|
| 2539 |
except Exception as e: # noqa: BLE001
|
| 2540 |
print(f"[aios-api] store resync: the Odoo source was not counted at mirror-pull time "
|
|
|
|
| 2810 |
# D-9131: count the source now, before the vendor pulls, so reconciliation compares the
|
| 2811 |
# mirror with the source as of the same moment. A pass without a good pull counts nothing.
|
| 2812 |
_source_counts = (_capture_resync_source_counts(_cad["tenant"])
|
| 2813 |
+
if _err is None and ((sync_report and not _manual)
|
| 2814 |
+
or (_manual and _odoo_requested)) else None)
|
| 2815 |
# ⭐ WAVE 27 item 17 (E's ASK ->A, resolved): whenever the Odoo RELATIONAL tables need
|
| 2816 |
# refreshing, they run AFTER the mirror they derive from. A confirmed zero-change report
|
| 2817 |
# is the W39 exception; deriving before a changed mirror advances would publish a one-cycle
|
api/registry_source.py
CHANGED
|
@@ -185,12 +185,36 @@ def generation(session, table_key):
|
|
| 185 |
if not callable(revision):
|
| 186 |
return None # Ordinary offline readers retain their original snapshot semantics.
|
| 187 |
tokens = {name: revision(name) for name in names}
|
| 188 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 189 |
|
| 190 |
|
| 191 |
def require_generation(session, table_key, before):
|
| 192 |
-
if before is
|
| 193 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 194 |
|
| 195 |
|
| 196 |
class BatchStore:
|
|
|
|
| 185 |
if not callable(revision):
|
| 186 |
return None # Ordinary offline readers retain their original snapshot semantics.
|
| 187 |
tokens = {name: revision(name) for name in names}
|
| 188 |
+
user = session.user if isinstance(session.user, dict) else {}
|
| 189 |
+
# A sign in or activity stamp never changes what the reader may see, so it never forces a rebuild.
|
| 190 |
+
policy = {key: value for key, value in user.items() if key not in USER_NOISE}
|
| 191 |
+
return copy.deepcopy((policy, tokens))
|
| 192 |
+
|
| 193 |
+
|
| 194 |
+
#: User record keys that change on sign in or use and never change what the reader may see.
|
| 195 |
+
USER_NOISE = frozenset(('last_login', 'last_active', 'updated_at', 'epoch', 'avatar', 'whatsapp'))
|
| 196 |
+
|
| 197 |
+
|
| 198 |
+
def moved_keys(before, after):
|
| 199 |
+
"""Sorted names of the generation parts that differ: store keys, plus `user` for the policy."""
|
| 200 |
+
if not (isinstance(before, tuple) and isinstance(after, tuple)
|
| 201 |
+
and len(before) == len(after) == 2):
|
| 202 |
+
return ['unknown']
|
| 203 |
+
moved = ['user'] if before[0] != after[0] else []
|
| 204 |
+
tokens_before, tokens_after = before[1] or {}, after[1] or {}
|
| 205 |
+
return moved + sorted(str(name) for name in set(tokens_before) | set(tokens_after)
|
| 206 |
+
if tokens_before.get(name) != tokens_after.get(name))
|
| 207 |
|
| 208 |
|
| 209 |
def require_generation(session, table_key, before):
|
| 210 |
+
if before is None:
|
| 211 |
+
return
|
| 212 |
+
after = generation(session, table_key)
|
| 213 |
+
if after != before:
|
| 214 |
+
exc = err(409, 'drill_population_changed', 'These records changed while loading. Try again.')
|
| 215 |
+
# The rebuild log names the moved keys. They never reach the client.
|
| 216 |
+
exc.moved = moved_keys(before, after)
|
| 217 |
+
raise exc
|
| 218 |
|
| 219 |
|
| 220 |
class BatchStore:
|
api/request_timing.py
CHANGED
|
@@ -70,25 +70,46 @@ class DeadlineCapacityExceeded(RuntimeError):
|
|
| 70 |
|
| 71 |
|
| 72 |
class _StatementAlarm:
|
|
|
|
|
|
|
|
|
|
| 73 |
def __init__(self, owner, connection, acc):
|
| 74 |
self.owner, self.connection, self.acc = owner, connection, acc
|
| 75 |
self.lock = threading.Lock()
|
| 76 |
self.active, self.expired = True, False
|
|
|
|
| 77 |
|
| 78 |
def fire(self):
|
| 79 |
with self.lock:
|
| 80 |
-
if
|
| 81 |
-
|
| 82 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 83 |
self.connection.interrupt()
|
|
|
|
|
|
|
|
|
|
| 84 |
|
| 85 |
def close(self):
|
| 86 |
# Serialize disarm with interrupt before the cursor can execute another statement.
|
| 87 |
with self.lock:
|
| 88 |
self.active = False
|
| 89 |
-
expired = self.expired
|
| 90 |
with self.owner.lock:
|
| 91 |
self.owner.active.discard(self)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 92 |
return expired
|
| 93 |
|
| 94 |
|
|
|
|
| 70 |
|
| 71 |
|
| 72 |
class _StatementAlarm:
|
| 73 |
+
#: Repeat interrupt() this often until disarm: DuckDB can lose a single one on a parameterized execute.
|
| 74 |
+
REFIRE = .05
|
| 75 |
+
|
| 76 |
def __init__(self, owner, connection, acc):
|
| 77 |
self.owner, self.connection, self.acc = owner, connection, acc
|
| 78 |
self.lock = threading.Lock()
|
| 79 |
self.active, self.expired = True, False
|
| 80 |
+
self.fired_at, self.last, self.sent = None, 0.0, 0
|
| 81 |
|
| 82 |
def fire(self):
|
| 83 |
with self.lock:
|
| 84 |
+
if not self.active:
|
| 85 |
+
return
|
| 86 |
+
now = time.monotonic()
|
| 87 |
+
if not self.expired:
|
| 88 |
+
if not (self.acc.cancelled or now >= self.acc.deadline):
|
| 89 |
+
return
|
| 90 |
+
self.expired, self.fired_at = True, now
|
| 91 |
+
elif now - self.last < self.REFIRE:
|
| 92 |
+
return
|
| 93 |
+
self.last = now
|
| 94 |
+
self.sent += 1
|
| 95 |
+
try:
|
| 96 |
self.connection.interrupt()
|
| 97 |
+
except Exception:
|
| 98 |
+
self.active = False
|
| 99 |
+
raise
|
| 100 |
|
| 101 |
def close(self):
|
| 102 |
# Serialize disarm with interrupt before the cursor can execute another statement.
|
| 103 |
with self.lock:
|
| 104 |
self.active = False
|
| 105 |
+
expired, fired_at, sent = self.expired, self.fired_at, self.sent
|
| 106 |
with self.owner.lock:
|
| 107 |
self.owner.active.discard(self)
|
| 108 |
+
if expired and fired_at is not None:
|
| 109 |
+
now = time.monotonic()
|
| 110 |
+
LOG.warning("[deadline-interrupt] statement stopped %.0f ms after the first interrupt "
|
| 111 |
+
"(%d sent), %.0f ms past the request deadline", (now - fired_at) * 1000, sent,
|
| 112 |
+
(now - self.acc.deadline) * 1000)
|
| 113 |
return expired
|
| 114 |
|
| 115 |
|
api/routes_opportunity_finder.py
CHANGED
|
@@ -85,6 +85,12 @@ def _app_writer(app: str = Query(..., min_length=1), session: Session = Depends(
|
|
| 85 |
return session
|
| 86 |
|
| 87 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 88 |
def _with_spark(of, st, items):
|
| 89 |
"""Each page item gains `spark`: its rank in the newest SPARK_WEEKS held weeks, oldest first."""
|
| 90 |
weeks = of.weeks(None, st=st)[-SPARK_WEEKS:] if SPARK_WEEKS > 0 else []
|
|
@@ -102,10 +108,12 @@ def app_screen(app: str = Query(..., min_length=1), status: str | None = Query(N
|
|
| 102 |
of = _module()
|
| 103 |
record = _app(session, app)
|
| 104 |
st = session.runtime
|
| 105 |
-
listed = _call(of.items, _tenant(session), session.uname, status=status, cursor=cursor, st=st
|
|
|
|
| 106 |
listed["items"] = _call(_with_spark, of, st, listed["items"])
|
| 107 |
return {"canEdit": _may_edit(session, record), "week": listed["week"], "priorWeek": listed["priorWeek"],
|
| 108 |
-
"weeks": listed["weeks"], "
|
|
|
|
| 109 |
"total": listed["total"], "items": listed["items"], "cursor": listed["cursor"],
|
| 110 |
"problems": listed["problems"], "statuses": list(of.STATUSES),
|
| 111 |
"settings": _call(of.settings, _tenant(session), st=st)}
|
|
@@ -114,7 +122,8 @@ def app_screen(app: str = Query(..., min_length=1), status: str | None = Query(N
|
|
| 114 |
@router.get("/items/{item_id}")
|
| 115 |
def item_detail(item_id: str, session: Session = Depends(_app_reader)):
|
| 116 |
of = _module()
|
| 117 |
-
return _call(of.item_detail, _tenant(session), session.uname, item_id, st=session.runtime
|
|
|
|
| 118 |
|
| 119 |
|
| 120 |
@router.patch("/items/{item_id}")
|
|
@@ -123,14 +132,14 @@ def update_item(item_id: str, status: str = Body(...), note: str | None = Body(N
|
|
| 123 |
session: Session = Depends(_app_writer)):
|
| 124 |
of = _module()
|
| 125 |
return _call(of.set_status, _tenant(session), _user(session), item_id, status, note=note, owner=owner,
|
| 126 |
-
due=due, st=session.runtime)
|
| 127 |
|
| 128 |
|
| 129 |
@router.get("/terms/{item_id}")
|
| 130 |
def term_trend(item_id: str, session: Session = Depends(_app_reader)):
|
| 131 |
"""A term's weekly rank trend and its newest top 3 clicked ASINs."""
|
| 132 |
of = _module()
|
| 133 |
-
return _call(of.term_detail, _tenant(session), item_id, st=session.runtime)
|
| 134 |
|
| 135 |
|
| 136 |
@router.get("/settings")
|
|
@@ -156,4 +165,5 @@ def write_settings(body: dict = Body(...), session: Session = Depends(_app_write
|
|
| 156 |
def validate(week: str | None = Query(None), start: str | None = Query(None), end: str | None = Query(None),
|
| 157 |
session: Session = Depends(_app_reader)):
|
| 158 |
of = _module()
|
| 159 |
-
return _call(of.validate, _tenant(session), week=week, start=start, end=end, st=session.runtime
|
|
|
|
|
|
| 85 |
return session
|
| 86 |
|
| 87 |
|
| 88 |
+
def _mirror_opener(session):
|
| 89 |
+
"""The typed `ut_amzn_*` door's cursor factory: tenant asserted, serving mirror version. The module closes it."""
|
| 90 |
+
import routes_odoo_tables
|
| 91 |
+
return lambda: routes_odoo_tables._table_cursor(session.runtime, SOURCE)
|
| 92 |
+
|
| 93 |
+
|
| 94 |
def _with_spark(of, st, items):
|
| 95 |
"""Each page item gains `spark`: its rank in the newest SPARK_WEEKS held weeks, oldest first."""
|
| 96 |
weeks = of.weeks(None, st=st)[-SPARK_WEEKS:] if SPARK_WEEKS > 0 else []
|
|
|
|
| 108 |
of = _module()
|
| 109 |
record = _app(session, app)
|
| 110 |
st = session.runtime
|
| 111 |
+
listed = _call(of.items, _tenant(session), session.uname, status=status, cursor=cursor, st=st,
|
| 112 |
+
opener=_mirror_opener(session))
|
| 113 |
listed["items"] = _call(_with_spark, of, st, listed["items"])
|
| 114 |
return {"canEdit": _may_edit(session, record), "week": listed["week"], "priorWeek": listed["priorWeek"],
|
| 115 |
+
"weeks": listed["weeks"], "windows": listed["windows"], "periodType": listed["periodType"],
|
| 116 |
+
"departments": listed["departments"], "counts": listed["counts"],
|
| 117 |
"total": listed["total"], "items": listed["items"], "cursor": listed["cursor"],
|
| 118 |
"problems": listed["problems"], "statuses": list(of.STATUSES),
|
| 119 |
"settings": _call(of.settings, _tenant(session), st=st)}
|
|
|
|
| 122 |
@router.get("/items/{item_id}")
|
| 123 |
def item_detail(item_id: str, session: Session = Depends(_app_reader)):
|
| 124 |
of = _module()
|
| 125 |
+
return _call(of.item_detail, _tenant(session), session.uname, item_id, st=session.runtime,
|
| 126 |
+
opener=_mirror_opener(session))
|
| 127 |
|
| 128 |
|
| 129 |
@router.patch("/items/{item_id}")
|
|
|
|
| 132 |
session: Session = Depends(_app_writer)):
|
| 133 |
of = _module()
|
| 134 |
return _call(of.set_status, _tenant(session), _user(session), item_id, status, note=note, owner=owner,
|
| 135 |
+
due=due, st=session.runtime, opener=_mirror_opener(session))
|
| 136 |
|
| 137 |
|
| 138 |
@router.get("/terms/{item_id}")
|
| 139 |
def term_trend(item_id: str, session: Session = Depends(_app_reader)):
|
| 140 |
"""A term's weekly rank trend and its newest top 3 clicked ASINs."""
|
| 141 |
of = _module()
|
| 142 |
+
return _call(of.term_detail, _tenant(session), item_id, st=session.runtime, opener=_mirror_opener(session))
|
| 143 |
|
| 144 |
|
| 145 |
@router.get("/settings")
|
|
|
|
| 165 |
def validate(week: str | None = Query(None), start: str | None = Query(None), end: str | None = Query(None),
|
| 166 |
session: Session = Depends(_app_reader)):
|
| 167 |
of = _module()
|
| 168 |
+
return _call(of.validate, _tenant(session), week=week, start=start, end=end, st=session.runtime,
|
| 169 |
+
opener=_mirror_opener(session))
|
api/routes_sheet.py
CHANGED
|
@@ -266,15 +266,38 @@ def _row_scope_applies(session, database):
|
|
| 266 |
|
| 267 |
def _registry_mirror_settled(session, cur, table_key, metric_window, needed):
|
| 268 |
"""Rebuild when the source changed during the build. Only the last change refuses."""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 269 |
for attempt in range(_MIRROR_BUILD_TRIES):
|
|
|
|
|
|
|
| 270 |
try:
|
| 271 |
return _registry_mirror(session, cur, table_key, metric_window, needed=needed)
|
| 272 |
except HTTPException as exc:
|
| 273 |
detail = exc.detail if isinstance(exc.detail, dict) else {}
|
| 274 |
-
if (
|
| 275 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 276 |
raise
|
| 277 |
-
print(f"[sheet] mirror {table_key} changed during its build, rebuild
|
|
|
|
| 278 |
flush=True)
|
| 279 |
|
| 280 |
|
|
@@ -4168,9 +4191,19 @@ def _refresh_desk_run(session, row):
|
|
| 4168 |
return sheets.refresh_scope(session.runtime, workbook, args.get("scope"), session)
|
| 4169 |
with _sheet_request_scope(session):
|
| 4170 |
if args.get("kind") == "retry":
|
| 4171 |
-
|
| 4172 |
-
|
| 4173 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4174 |
|
| 4175 |
|
| 4176 |
# ── the opening cache: a repeat open of the same tab at the same state is served from memory ─────
|
|
@@ -6577,7 +6610,9 @@ def _compute_cell_scoped(workbook_id: str, addr: str, body, session: Session):
|
|
| 6577 |
def view_rows(workbook_id: str, body: object = Body(default=None),
|
| 6578 |
session: Session = Depends(require_session)):
|
| 6579 |
"""W62 C2 / R29: read the live rows of a placed view under this caller's walls."""
|
|
|
|
| 6580 |
from fastapi.responses import JSONResponse
|
|
|
|
| 6581 |
from core import perm_scope, sheets, user_tables
|
| 6582 |
from modules import sheet as sheet_mod
|
| 6583 |
|
|
@@ -6603,74 +6638,74 @@ def view_rows(workbook_id: str, body: object = Body(default=None),
|
|
| 6603 |
|
| 6604 |
# W62 C2: "Filter and sort apply to ALL rows BEFORE limit. total counts every matching row."
|
| 6605 |
_refuse_unapplied_audience(session, database)
|
| 6606 |
-
|
| 6607 |
-
if perm_scope.row_scope_applies(session.user, database) else None)
|
| 6608 |
-
wall_nodes, wall_conj = _fe_parts(wall_tree)
|
| 6609 |
-
scope_nodes, scope_conj = _fe_parts(binding.get("filter"))
|
| 6610 |
-
scope_nodes = _rewrite_related_filters(session, database, scope_nodes)
|
| 6611 |
# W62 C2: a filter-only field obeys the same grants; linked fields have their own resolver wall.
|
| 6612 |
for key in sorted(sheet_mod.filter_columns_named(scope_nodes)):
|
| 6613 |
if not key.startswith(sheet_mod.RELATED_FILTER_PREFIX) and key not in visible:
|
| 6614 |
return JSONResponse(status_code=400, content={
|
| 6615 |
"error": "view_field_unknown", "message": f"Field {key} is not available in this database."})
|
| 6616 |
today = _sheet_today(session)
|
|
|
|
|
|
|
| 6617 |
try:
|
| 6618 |
-
scope_nodes
|
| 6619 |
-
|
| 6620 |
-
|
| 6621 |
-
|
| 6622 |
-
|
| 6623 |
-
|
| 6624 |
-
|
| 6625 |
-
|
| 6626 |
-
|
| 6627 |
-
|
| 6628 |
-
|
| 6629 |
-
|
| 6630 |
-
|
| 6631 |
-
|
| 6632 |
-
|
| 6633 |
-
|
| 6634 |
-
|
| 6635 |
-
|
| 6636 |
-
|
| 6637 |
-
|
| 6638 |
-
|
| 6639 |
-
|
| 6640 |
-
|
| 6641 |
-
|
| 6642 |
-
|
| 6643 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6644 |
except sheet_mod.SheetRefusal as exc:
|
| 6645 |
return JSONResponse(status_code=400, content={"error": exc.code, "message": exc.message})
|
| 6646 |
except ValueError:
|
| 6647 |
raise err(409, "scope_not_applicable",
|
| 6648 |
"This view uses a restriction that cannot be applied to this database's live connection.")
|
| 6649 |
-
|
| 6650 |
-
# W62 C2 / standing rule 1: "limit caps what is PLACED"; the matching count is unlimited.
|
| 6651 |
-
total = (cur.execute(count_sql, params).fetchone() or [0])[0]
|
| 6652 |
-
# ⭐⭐ W62-T14 — EVERY ROW CARRIES ITS RECORD ID, ALWAYS, AND IT IS NOT A PROJECTED FIELD.
|
| 6653 |
-
# A placed Grid over one of our own tables writes a record back, and the write door is Data's
|
| 6654 |
-
# own `PATCH /api/v1/tables/{database}/rows/{pid}`: without the id the client can only guess
|
| 6655 |
-
# from a row's POSITION in the sheet, which is wrong the moment anyone sorts, filters or
|
| 6656 |
-
# refreshes. So the id rides beside the values under the key `pid` while `fields` still names
|
| 6657 |
-
# only the columns the sheet paints. ⛔ It is the SAME `spec["id"]` expression `build_ids_query`
|
| 6658 |
-
# uses, so the identity here and the identity the drill lists can never be two different things.
|
| 6659 |
-
id_expr = str(spec.get("id") or "").strip()
|
| 6660 |
-
projection = ", ".join([*(cols[key]["sql"] for key in requested),
|
| 6661 |
-
*([id_expr] if id_expr else [])])
|
| 6662 |
-
rows_sql = count_sql.replace("SELECT COUNT(*) FROM", f"SELECT {projection} FROM", 1)
|
| 6663 |
-
if binding["sort"]:
|
| 6664 |
-
# W62 C2 / R30: expressions come only from resolved columns; directions are validated.
|
| 6665 |
-
order = ", ".join(cols[item["field"]]["sql"] + (" ASC" if item["dir"] == "asc" else " DESC")
|
| 6666 |
-
for item in binding["sort"])
|
| 6667 |
-
rows_sql += " ORDER BY " + order
|
| 6668 |
-
rows_sql += f" LIMIT {binding['limit']}"
|
| 6669 |
-
keys = [*requested, *(["pid"] if id_expr else [])]
|
| 6670 |
-
rows = [dict(zip(keys, row)) for row in cur.execute(rows_sql, params).fetchall()]
|
| 6671 |
# W62 C2 / R34: Data's _records_or_refuse uses database access (checked above) plus
|
| 6672 |
# W62 R34: records_mutable, with no separate edit-role grant; its row/field walls still apply.
|
| 6673 |
editable = user_tables.records_mutable(database, st=session.runtime)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6674 |
return {"fields": [{"key": key, "label": visible[key].get("label") or key,
|
| 6675 |
"kind": visible[key].get("type") or visible[key].get("kind") or "text"}
|
| 6676 |
for key in requested],
|
|
@@ -6678,6 +6713,94 @@ def view_rows(workbook_id: str, body: object = Body(default=None),
|
|
| 6678 |
"viewType": binding["viewType"], "editable": bool(editable)}
|
| 6679 |
|
| 6680 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6681 |
_DRILL_MIRRORS = OrderedDict()
|
| 6682 |
# The registry guard: held only for lookups and bookkeeping, never across a build or a yield.
|
| 6683 |
_DRILL_MIRROR_LOCK = threading.RLock()
|
|
|
|
| 266 |
|
| 267 |
def _registry_mirror_settled(session, cur, table_key, metric_window, needed):
|
| 268 |
"""Rebuild when the source changed during the build. Only the last change refuses."""
|
| 269 |
+
from copy import deepcopy
|
| 270 |
+
from time import perf_counter
|
| 271 |
+
from core.request_timing import remaining_budget
|
| 272 |
+
slot = ("mirror_refused", table_key, json.dumps(metric_window, sort_keys=True, default=str))
|
| 273 |
+
memo = getattr(session, "_sheet_read_memo", None)
|
| 274 |
+
refused = memo if isinstance(memo, dict) else {}
|
| 275 |
+
if slot in refused:
|
| 276 |
+
# ⛔ One refused build per database and window per request: every further cell would rebuild.
|
| 277 |
+
status, detail = refused[slot]
|
| 278 |
+
raise HTTPException(status_code=status, detail=deepcopy(detail))
|
| 279 |
for attempt in range(_MIRROR_BUILD_TRIES):
|
| 280 |
+
started = perf_counter()
|
| 281 |
+
session._sheet_mirror_builds = getattr(session, "_sheet_mirror_builds", 0) + 1
|
| 282 |
try:
|
| 283 |
return _registry_mirror(session, cur, table_key, metric_window, needed=needed)
|
| 284 |
except HTTPException as exc:
|
| 285 |
detail = exc.detail if isinstance(exc.detail, dict) else {}
|
| 286 |
+
if (detail.get("error") or {}).get("code") != "drill_population_changed":
|
| 287 |
+
raise
|
| 288 |
+
spent = perf_counter() - started
|
| 289 |
+
moved = ",".join(getattr(exc, "moved", None) or ["unknown"])
|
| 290 |
+
left = remaining_budget()
|
| 291 |
+
# A rebuild that cannot finish inside the budget only turns a held cell into a timeout.
|
| 292 |
+
short = left is not None and left < spent * 1.2
|
| 293 |
+
if attempt + 1 >= _MIRROR_BUILD_TRIES or short:
|
| 294 |
+
print(f"[sheet] mirror {table_key} changed during its build, refused "
|
| 295 |
+
f"moved={moved} build_ms={int(spent * 1000)}"
|
| 296 |
+
+ (f" budget_left_ms={int(left * 1000)}" if short else ""), flush=True)
|
| 297 |
+
refused[slot] = (exc.status_code, deepcopy(exc.detail))
|
| 298 |
raise
|
| 299 |
+
print(f"[sheet] mirror {table_key} changed during its build, rebuild "
|
| 300 |
+
f"{attempt + 2}/{_MIRROR_BUILD_TRIES} moved={moved} build_ms={int(spent * 1000)}",
|
| 301 |
flush=True)
|
| 302 |
|
| 303 |
|
|
|
|
| 4191 |
return sheets.refresh_scope(session.runtime, workbook, args.get("scope"), session)
|
| 4192 |
with _sheet_request_scope(session):
|
| 4193 |
if args.get("kind") == "retry":
|
| 4194 |
+
result = _refresh_sheet_scoped(workbook, session, retry_id=args.get("retryId"))
|
| 4195 |
+
else:
|
| 4196 |
+
only = set(args.get("cells") or ()) if args.get("kind") == "cells" else None
|
| 4197 |
+
result = _refresh_sheet_scoped(workbook, session, only=only)
|
| 4198 |
+
if isinstance(result, dict) and isinstance(result.get("workbook"), dict):
|
| 4199 |
+
# Placed views read their rows after the run ends. Their mirror builds here, in a fresh
|
| 4200 |
+
# scope, so the view-rows call reuses it. The run's own result never depends on it.
|
| 4201 |
+
try:
|
| 4202 |
+
with _sheet_request_scope(session):
|
| 4203 |
+
_prewarm_view_mirrors(session, result["workbook"])
|
| 4204 |
+
except Exception as exc: # noqa: BLE001
|
| 4205 |
+
print(f"[sheet] view prewarm wb={workbook} failed {type(exc).__name__}", flush=True)
|
| 4206 |
+
return result
|
| 4207 |
|
| 4208 |
|
| 4209 |
# ── the opening cache: a repeat open of the same tab at the same state is served from memory ─────
|
|
|
|
| 6610 |
def view_rows(workbook_id: str, body: object = Body(default=None),
|
| 6611 |
session: Session = Depends(require_session)):
|
| 6612 |
"""W62 C2 / R29: read the live rows of a placed view under this caller's walls."""
|
| 6613 |
+
from time import perf_counter
|
| 6614 |
from fastapi.responses import JSONResponse
|
| 6615 |
+
import odoo_relational as rel
|
| 6616 |
from core import perm_scope, sheets, user_tables
|
| 6617 |
from modules import sheet as sheet_mod
|
| 6618 |
|
|
|
|
| 6638 |
|
| 6639 |
# W62 C2: "Filter and sort apply to ALL rows BEFORE limit. total counts every matching row."
|
| 6640 |
_refuse_unapplied_audience(session, database)
|
| 6641 |
+
wall_nodes, wall_conj, scope_nodes, scope_conj = _view_scope(session, database, binding)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6642 |
# W62 C2: a filter-only field obeys the same grants; linked fields have their own resolver wall.
|
| 6643 |
for key in sorted(sheet_mod.filter_columns_named(scope_nodes)):
|
| 6644 |
if not key.startswith(sheet_mod.RELATED_FILTER_PREFIX) and key not in visible:
|
| 6645 |
return JSONResponse(status_code=400, content={
|
| 6646 |
"error": "view_field_unknown", "message": f"Field {key} is not available in this database."})
|
| 6647 |
today = _sheet_today(session)
|
| 6648 |
+
started = perf_counter()
|
| 6649 |
+
builds = getattr(session, "_sheet_mirror_builds", 0)
|
| 6650 |
try:
|
| 6651 |
+
scope_nodes, window, related_bags, door = _view_mirror_door(
|
| 6652 |
+
session, book, binding, database, scope_nodes, wall_nodes, today)
|
| 6653 |
+
with door as (made, _token):
|
| 6654 |
+
cur, spec, cols = made
|
| 6655 |
+
cols = dict(cols)
|
| 6656 |
+
if spec.get("scope_applied"):
|
| 6657 |
+
wall_nodes, wall_conj = [], "and"
|
| 6658 |
+
related = related_bags(cur, spec, cols, window=window)
|
| 6659 |
+
for key in dict.fromkeys([*requested, *sorted(sorted_keys | sheet_mod.filter_columns_named(scope_nodes))]):
|
| 6660 |
+
if key.startswith(sheet_mod.RELATED_FILTER_PREFIX):
|
| 6661 |
+
continue
|
| 6662 |
+
if key in hidden or not (cols.get(key) or {}).get("sql"):
|
| 6663 |
+
return JSONResponse(status_code=400, content={
|
| 6664 |
+
"error": "view_field_unknown", "message": f"Field {key} is not available in this database."})
|
| 6665 |
+
# W62 C2: an unanswerable row wall must refuse before either SQL read can widen it.
|
| 6666 |
+
wall_compiled = sheet_mod.compile_scope(wall_nodes, wall_conj, cols, today=today)
|
| 6667 |
+
if sheet_mod.widen_limits(wall_nodes, cols, wall_compiled):
|
| 6668 |
+
raise ValueError("unapplied row wall")
|
| 6669 |
+
count_sql, params, limits = sheet_mod.build_aggregate_query(
|
| 6670 |
+
spec, {"aggregate": {"fn": "count"}}, cols, scope_nodes=scope_nodes,
|
| 6671 |
+
scope_conj=scope_conj, wall_nodes=wall_nodes, wall_conj=wall_conj, today=today,
|
| 6672 |
+
related=related, summary_default=False)
|
| 6673 |
+
if limits:
|
| 6674 |
+
raise ValueError("unapplied view filter")
|
| 6675 |
+
|
| 6676 |
+
# W62 C2 / standing rule 1: "limit caps what is PLACED"; the matching count is unlimited.
|
| 6677 |
+
total = (cur.execute(count_sql, params).fetchone() or [0])[0]
|
| 6678 |
+
# ⭐⭐ W62-T14 — EVERY ROW CARRIES ITS RECORD ID, ALWAYS, AND IT IS NOT A PROJECTED FIELD.
|
| 6679 |
+
# A placed Grid over one of our own tables writes a record back, and the write door is Data's
|
| 6680 |
+
# own `PATCH /api/v1/tables/{database}/rows/{pid}`: without the id the client can only guess
|
| 6681 |
+
# from a row's POSITION in the sheet, which is wrong the moment anyone sorts, filters or
|
| 6682 |
+
# refreshes. So the id rides beside the values under the key `pid` while `fields` still names
|
| 6683 |
+
# only the columns the sheet paints. ⛔ It is the SAME `spec["id"]` expression `build_ids_query`
|
| 6684 |
+
# uses, so the identity here and the identity the drill lists can never be two different things.
|
| 6685 |
+
id_expr = str(spec.get("id") or "").strip()
|
| 6686 |
+
projection = ", ".join([*(cols[key]["sql"] for key in requested),
|
| 6687 |
+
*([id_expr] if id_expr else [])])
|
| 6688 |
+
rows_sql = count_sql.replace("SELECT COUNT(*) FROM", f"SELECT {projection} FROM", 1)
|
| 6689 |
+
if binding["sort"]:
|
| 6690 |
+
# W62 C2 / R30: expressions come only from resolved columns; directions are validated.
|
| 6691 |
+
order = ", ".join(cols[item["field"]]["sql"] + (" ASC" if item["dir"] == "asc" else " DESC")
|
| 6692 |
+
for item in binding["sort"])
|
| 6693 |
+
rows_sql += " ORDER BY " + order
|
| 6694 |
+
rows_sql += f" LIMIT {binding['limit']}"
|
| 6695 |
+
keys = [*requested, *(["pid"] if id_expr else [])]
|
| 6696 |
+
rows = [dict(zip(keys, row)) for row in cur.execute(rows_sql, params).fetchall()]
|
| 6697 |
except sheet_mod.SheetRefusal as exc:
|
| 6698 |
return JSONResponse(status_code=400, content={"error": exc.code, "message": exc.message})
|
| 6699 |
except ValueError:
|
| 6700 |
raise err(409, "scope_not_applicable",
|
| 6701 |
"This view uses a restriction that cannot be applied to this database's live connection.")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6702 |
# W62 C2 / R34: Data's _records_or_refuse uses database access (checked above) plus
|
| 6703 |
# W62 R34: records_mutable, with no separate edit-role grant; its row/field walls still apply.
|
| 6704 |
editable = user_tables.records_mutable(database, st=session.runtime)
|
| 6705 |
+
reused = (int(getattr(session, "_sheet_mirror_builds", 0) == builds)
|
| 6706 |
+
if database in rel._REGISTRY_GRID_FIELDS else "na")
|
| 6707 |
+
print(f"[sheet] view-rows wb={workbook_id} db={database} ms={int((perf_counter() - started) * 1000)} "
|
| 6708 |
+
f"reused={reused} rows={len(rows)} total={int(total or 0)}", flush=True)
|
| 6709 |
return {"fields": [{"key": key, "label": visible[key].get("label") or key,
|
| 6710 |
"kind": visible[key].get("type") or visible[key].get("kind") or "text"}
|
| 6711 |
for key in requested],
|
|
|
|
| 6713 |
"viewType": binding["viewType"], "editable": bool(editable)}
|
| 6714 |
|
| 6715 |
|
| 6716 |
+
def _view_scope(session, database, binding):
|
| 6717 |
+
"""`(wall_nodes, wall_conj, scope_nodes, scope_conj)` of one placed view for this reader."""
|
| 6718 |
+
import core.perm_scope as perm_scope
|
| 6719 |
+
_refuse_unapplied_audience(session, database)
|
| 6720 |
+
wall_tree = ((perm_scope.entry(session.user, database) or {}).get("filter")
|
| 6721 |
+
if perm_scope.row_scope_applies(session.user, database) else None)
|
| 6722 |
+
wall_nodes, wall_conj = _fe_parts(wall_tree)
|
| 6723 |
+
scope_nodes, scope_conj = _fe_parts(binding.get("filter"))
|
| 6724 |
+
return wall_nodes, wall_conj, _rewrite_related_filters(session, database, scope_nodes), scope_conj
|
| 6725 |
+
|
| 6726 |
+
|
| 6727 |
+
def _view_mirror_door(session, book, binding, database, scope_nodes, wall_nodes, today):
|
| 6728 |
+
"""`(scope_nodes, window, related_bags, door)`; `door` yields `((cur, spec, cols), token)`.
|
| 6729 |
+
|
| 6730 |
+
A compiled registry grid reads through the reader-keyed drill mirror cache, so a repeat view
|
| 6731 |
+
and a view the desk prewarmed reuse one build. Every other database builds per request.
|
| 6732 |
+
"""
|
| 6733 |
+
import odoo_relational as rel
|
| 6734 |
+
from modules import sheet as sheet_mod
|
| 6735 |
+
scope_nodes = _filter_cell_values(session, scope_nodes, book=book, today=today)
|
| 6736 |
+
window = _ending_window(session, binding, book=book, today=today)
|
| 6737 |
+
needed = (set(binding["fields"]) | {item["field"] for item in binding["sort"]}
|
| 6738 |
+
| sheet_mod.filter_columns_named(scope_nodes) | sheet_mod.filter_columns_named(wall_nodes))
|
| 6739 |
+
related_bags = _related_leaf_bags(session, database, scope_nodes, needed)
|
| 6740 |
+
if database in rel._REGISTRY_GRID_FIELDS:
|
| 6741 |
+
databases = {database}
|
| 6742 |
+
databases.update(_binding_related_entry(session, database, via)["target"]
|
| 6743 |
+
for via in sheet_mod.related_vias(scope_nodes))
|
| 6744 |
+
return scope_nodes, window, related_bags, _drill_mirror(session, database, window, needed, databases)
|
| 6745 |
+
made = _mirror(session)(database, metric_window=window, needed=needed)
|
| 6746 |
+
return scope_nodes, window, related_bags, contextlib.nullcontext((made, None))
|
| 6747 |
+
|
| 6748 |
+
|
| 6749 |
+
#: A desk prewarm of placed views starts only with this much of the run's budget left.
|
| 6750 |
+
VIEW_PREWARM_MIN_SECONDS = 90.0
|
| 6751 |
+
#: And it stops this long before the run's own deadline.
|
| 6752 |
+
VIEW_PREWARM_MARGIN_SECONDS = 20.0
|
| 6753 |
+
|
| 6754 |
+
|
| 6755 |
+
def _prewarm_view_mirrors(session, book):
|
| 6756 |
+
"""Build the drill mirror each registry-grid placed view of `book` reads, as this reader.
|
| 6757 |
+
|
| 6758 |
+
The client reads view rows after a Refresh within a 45 s request budget. A customer grid
|
| 6759 |
+
build costs more than that, so the desk builds it first and the view-rows call reuses it.
|
| 6760 |
+
A failed prewarm leaves the view to build on its own request.
|
| 6761 |
+
"""
|
| 6762 |
+
import time
|
| 6763 |
+
from copy import deepcopy
|
| 6764 |
+
from time import perf_counter
|
| 6765 |
+
import odoo_relational as rel
|
| 6766 |
+
from core import request_timing, sheets
|
| 6767 |
+
left = request_timing.remaining_budget()
|
| 6768 |
+
warmed = 0
|
| 6769 |
+
for tab in book.get("sheets") or []:
|
| 6770 |
+
for addr, cell in (tab.get("cells") or {}).items():
|
| 6771 |
+
if not (isinstance(cell, dict) and cell.get("kind") == "bound" and _is_view_binding(cell)
|
| 6772 |
+
and str((cell.get("binding") or {}).get("database") or "") in rel._REGISTRY_GRID_FIELDS):
|
| 6773 |
+
continue
|
| 6774 |
+
if left is not None and left < VIEW_PREWARM_MIN_SECONDS:
|
| 6775 |
+
print(f"[sheet] view prewarm wb={book.get('id')} skipped budget_left_ms={int(left * 1000)}",
|
| 6776 |
+
flush=True)
|
| 6777 |
+
return warmed
|
| 6778 |
+
started = perf_counter()
|
| 6779 |
+
builds = getattr(session, "_sheet_mirror_builds", 0)
|
| 6780 |
+
try:
|
| 6781 |
+
binding = sheets.clean_view_binding(deepcopy(cell["binding"]))
|
| 6782 |
+
database = _require_database(session, binding["database"])
|
| 6783 |
+
_refuse_unapplied_audience(session, database)
|
| 6784 |
+
wall_nodes, _wall_conj, scope_nodes, _conj = _view_scope(session, database, binding)
|
| 6785 |
+
deadline = (None if left is None else
|
| 6786 |
+
time.monotonic() + left - VIEW_PREWARM_MARGIN_SECONDS)
|
| 6787 |
+
with (request_timing.budget_scope(deadline) if deadline is not None
|
| 6788 |
+
else contextlib.nullcontext()):
|
| 6789 |
+
_nodes, window, _bags, door = _view_mirror_door(
|
| 6790 |
+
session, book, binding, database, scope_nodes, wall_nodes, _sheet_today(session))
|
| 6791 |
+
# A second view of the same reader and window reuses the first build.
|
| 6792 |
+
with door as (_made, token):
|
| 6793 |
+
outcome = "cached" if token else "uncached"
|
| 6794 |
+
warmed += 1 if token else 0
|
| 6795 |
+
except Exception as exc: # noqa: BLE001
|
| 6796 |
+
outcome = "failed " + type(exc).__name__
|
| 6797 |
+
print(f"[sheet] view prewarm wb={book.get('id')} cell={tab.get('id')}!{addr} {outcome} "
|
| 6798 |
+
f"built={int(getattr(session, '_sheet_mirror_builds', 0) != builds)} "
|
| 6799 |
+
f"ms={int((perf_counter() - started) * 1000)}", flush=True)
|
| 6800 |
+
left = request_timing.remaining_budget()
|
| 6801 |
+
return warmed
|
| 6802 |
+
|
| 6803 |
+
|
| 6804 |
_DRILL_MIRRORS = OrderedDict()
|
| 6805 |
# The registry guard: held only for lookups and bookkeeping, never across a build or a yield.
|
| 6806 |
_DRILL_MIRROR_LOCK = threading.RLock()
|
platform/evals/royal_questions.yml
CHANGED
|
@@ -91,7 +91,7 @@ questions:
|
|
| 91 |
- id: l03
|
| 92 |
kind: list
|
| 93 |
ask: Past due receivables by customer
|
| 94 |
-
oracle: {topic:
|
| 95 |
- id: l04
|
| 96 |
kind: list
|
| 97 |
ask: Payables balance by vendor
|
|
|
|
| 91 |
- id: l03
|
| 92 |
kind: list
|
| 93 |
ask: Past due receivables by customer
|
| 94 |
+
oracle: {topic: gl_lines, measures: [ar_overdue], group: [partner], filters: [], scope: receivable_lines}
|
| 95 |
- id: l04
|
| 96 |
kind: list
|
| 97 |
ask: Payables balance by vendor
|
platform/harness/amazon_store.py
CHANGED
|
@@ -255,6 +255,8 @@ FIRST_LOAD_ROWS_PER_PASS = 3_000_000
|
|
| 255 |
BACKFILL_FLOOR_PREFIX = "sync_floor_"
|
| 256 |
#: Report windows whose stored rows cannot be healed in place; the next pass downloads them again.
|
| 257 |
RESYNC_PREFIX = "resync_"
|
|
|
|
|
|
|
| 258 |
FIRST_LOAD_END_KEY = "sync_first_load_end"
|
| 259 |
FIRST_LOAD_STATUS_KEY = "sync_first_load_status"
|
| 260 |
FIRST_LOAD_DUE_KEY = "sync_first_load_next_due"
|
|
@@ -3288,14 +3290,19 @@ def _queue_resync(con, mirror, types, where):
|
|
| 3288 |
return rows, windows
|
| 3289 |
|
| 3290 |
|
| 3291 |
-
def
|
| 3292 |
-
"""
|
| 3293 |
saved = _read_state(con, entity_key(RESYNC_PREFIX + table))
|
| 3294 |
try:
|
| 3295 |
-
start, end
|
| 3296 |
-
return _utc(start), _utc(end)
|
| 3297 |
except (TypeError, ValueError, KeyError, IndexError, AttributeError):
|
| 3298 |
-
return
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3299 |
|
| 3300 |
|
| 3301 |
def resync_landed(con, table, window):
|
|
@@ -5318,6 +5325,12 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
|
|
| 5318 |
if (mark.get("state") == "refused" and str(mark.get("detail") or "").startswith("reconciliation refused")
|
| 5319 |
and not _interval_reason(table, ticks.get(table), end)):
|
| 5320 |
ticks.update({name: None for name in (table, pairs.get(table)) if name in want})
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5321 |
plan = sync_plan(ticks, want, now=end)
|
| 5322 |
want = plan["tables"]
|
| 5323 |
for table, reason in plan["deferred"].items():
|
|
@@ -5538,6 +5551,7 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
|
|
| 5538 |
sibling_written, sibling_cleared = 0, False
|
| 5539 |
written, cols, cleared = 0, [], False
|
| 5540 |
landed_window = False
|
|
|
|
| 5541 |
pricing_asins = []
|
| 5542 |
operations = {}
|
| 5543 |
if first_load:
|
|
@@ -5625,12 +5639,14 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
|
|
| 5625 |
margin_windows = _month_windows(plan["start"], plan["end"])
|
| 5626 |
else:
|
| 5627 |
margin_windows = _windows(plan["start"], plan["end"], spec.get("max_window_days", MAX_WINDOW_DAYS))
|
| 5628 |
-
resync =
|
| 5629 |
-
if resync
|
| 5630 |
-
#
|
| 5631 |
-
|
|
|
|
| 5632 |
plan["end"] = read_watermark(con, table)
|
| 5633 |
-
plan["cause"] = f"resync of
|
|
|
|
| 5634 |
log(f"{table}: {plan['cause']}")
|
| 5635 |
elif table in GROUP_A_TABLES and not first_load and len(margin_windows) > 1:
|
| 5636 |
# One due window per table tick respects the vendor floor. The watermark
|
|
@@ -5655,8 +5671,10 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
|
|
| 5655 |
{"state": "access", "operation": operation}, rt, report, log)
|
| 5656 |
count = _sync_margin_window(con, table, fields, w0, w1,
|
| 5657 |
max_wait=max_wait, log=log)
|
| 5658 |
-
if
|
| 5659 |
-
|
|
|
|
|
|
|
| 5660 |
probe.finish(count)
|
| 5661 |
spend_rows(count)
|
| 5662 |
written += count
|
|
@@ -5670,6 +5688,9 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
|
|
| 5670 |
if first_load and w0 is not None:
|
| 5671 |
record_backfill_floor(con, table, w0)
|
| 5672 |
floors[table] = w0
|
|
|
|
|
|
|
|
|
|
| 5673 |
elif table != "settlements":
|
| 5674 |
period_spec = cx.REPORT_TYPES[table]
|
| 5675 |
windows = (([(task_start, task_end)] if task_end is not None else []) if first_load
|
|
@@ -5867,6 +5888,8 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri
|
|
| 5867 |
"rows": total_rows, "columns": column_count, "provenance": prov,
|
| 5868 |
"report": rtype, "window": plan["cause"], "incremental": plan["incremental"],
|
| 5869 |
"operations": operations}
|
|
|
|
|
|
|
| 5870 |
_record_state(con, key, "done", total_rows)
|
| 5871 |
_record_state(con, entity_key("sync_tick_" + table), "done", total_rows)
|
| 5872 |
if not any(v["state"] in ("refused", "throttled") for v in operations.values()):
|
|
@@ -7984,6 +8007,33 @@ def group_a_selftest(log=print, check=None):
|
|
| 7984 |
and queued_resync(con, "long_term_storage_fees") is None
|
| 7985 |
and con.execute(f"SELECT _period_start, _period_end, count(*) FROM {fees} "
|
| 7986 |
"GROUP BY 1, 2 ORDER BY 1").fetchall() == stored)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 7987 |
finally:
|
| 7988 |
con.close()
|
| 7989 |
|
|
|
|
| 255 |
BACKFILL_FLOOR_PREFIX = "sync_floor_"
|
| 256 |
#: Report windows whose stored rows cannot be healed in place; the next pass downloads them again.
|
| 257 |
RESYNC_PREFIX = "resync_"
|
| 258 |
+
#: Queued resync windows one pass downloads at most: Amazon's createReport burst, less the pass's spend.
|
| 259 |
+
RESYNC_REPORTS_PER_PASS = FIRST_LOAD_REPORTS_PER_PASS
|
| 260 |
FIRST_LOAD_END_KEY = "sync_first_load_end"
|
| 261 |
FIRST_LOAD_STATUS_KEY = "sync_first_load_status"
|
| 262 |
FIRST_LOAD_DUE_KEY = "sync_first_load_next_due"
|
|
|
|
| 3290 |
return rows, windows
|
| 3291 |
|
| 3292 |
|
| 3293 |
+
def queued_resyncs(con, table):
|
| 3294 |
+
"""Every queued resync window of `table`, oldest first, as `(start, end)` datetimes."""
|
| 3295 |
saved = _read_state(con, entity_key(RESYNC_PREFIX + table))
|
| 3296 |
try:
|
| 3297 |
+
return [(_utc(start), _utc(end)) for start, end in json.loads(saved["detail"])["windows"]]
|
|
|
|
| 3298 |
except (TypeError, ValueError, KeyError, IndexError, AttributeError):
|
| 3299 |
+
return []
|
| 3300 |
+
|
| 3301 |
+
|
| 3302 |
+
def queued_resync(con, table):
|
| 3303 |
+
"""The oldest queued resync window of `table` as `(start, end)` datetimes, or None."""
|
| 3304 |
+
windows = queued_resyncs(con, table)
|
| 3305 |
+
return windows[0] if windows else None
|
| 3306 |
|
| 3307 |
|
| 3308 |
def resync_landed(con, table, window):
|
|
|
|
| 5325 |
if (mark.get("state") == "refused" and str(mark.get("detail") or "").startswith("reconciliation refused")
|
| 5326 |
and not _interval_reason(table, ticks.get(table), end)):
|
| 5327 |
ticks.update({name: None for name in (table, pairs.get(table)) if name in want})
|
| 5328 |
+
for table in want:
|
| 5329 |
+
tick = ticks.get(table)
|
| 5330 |
+
# A queued resync is a repair: only Amazon's own floor and the stagger hold it, never the daily floor.
|
| 5331 |
+
if (tick and tick["state"] == "done" and queued_resync(con, table) is not None
|
| 5332 |
+
and read_watermark(con, table) and not _interval_reason(table, tick, end)):
|
| 5333 |
+
ticks[table] = {**tick, "state": "resync"}
|
| 5334 |
plan = sync_plan(ticks, want, now=end)
|
| 5335 |
want = plan["tables"]
|
| 5336 |
for table, reason in plan["deferred"].items():
|
|
|
|
| 5551 |
sibling_written, sibling_cleared = 0, False
|
| 5552 |
written, cols, cleared = 0, [], False
|
| 5553 |
landed_window = False
|
| 5554 |
+
resync, resync_count, resync_left = [], 0, None
|
| 5555 |
pricing_asins = []
|
| 5556 |
operations = {}
|
| 5557 |
if first_load:
|
|
|
|
| 5639 |
margin_windows = _month_windows(plan["start"], plan["end"])
|
| 5640 |
else:
|
| 5641 |
margin_windows = _windows(plan["start"], plan["end"], spec.get("max_window_days", MAX_WINDOW_DAYS))
|
| 5642 |
+
resync = [] if first_load or spec.get("snapshot") else queued_resyncs(con, table)
|
| 5643 |
+
if resync and read_watermark(con, table):
|
| 5644 |
+
# Queued windows take this tick, several within the quota left; the watermark stays.
|
| 5645 |
+
budget = report_budget if isinstance(report_budget, int) else RESYNC_REPORTS_PER_PASS
|
| 5646 |
+
margin_windows = resync[:max(1, budget - documents)]
|
| 5647 |
plan["end"] = read_watermark(con, table)
|
| 5648 |
+
plan["cause"] = (f"resync of {len(margin_windows)} stored windows "
|
| 5649 |
+
f"from {_iso(margin_windows[0][0])}")
|
| 5650 |
log(f"{table}: {plan['cause']}")
|
| 5651 |
elif table in GROUP_A_TABLES and not first_load and len(margin_windows) > 1:
|
| 5652 |
# One due window per table tick respects the vendor floor. The watermark
|
|
|
|
| 5671 |
{"state": "access", "operation": operation}, rt, report, log)
|
| 5672 |
count = _sync_margin_window(con, table, fields, w0, w1,
|
| 5673 |
max_wait=max_wait, log=log)
|
| 5674 |
+
if (w0, w1) in resync:
|
| 5675 |
+
# Per window, so a failure later in this tick never downloads a landed window again.
|
| 5676 |
+
resync_landed(con, table, (w0, w1))
|
| 5677 |
+
resync_count += 1
|
| 5678 |
probe.finish(count)
|
| 5679 |
spend_rows(count)
|
| 5680 |
written += count
|
|
|
|
| 5688 |
if first_load and w0 is not None:
|
| 5689 |
record_backfill_floor(con, table, w0)
|
| 5690 |
floors[table] = w0
|
| 5691 |
+
if resync_count:
|
| 5692 |
+
resync_left = len(queued_resyncs(con, table))
|
| 5693 |
+
log(f"Amazon resync {table}: {resync_count} windows landed, {resync_left} left")
|
| 5694 |
elif table != "settlements":
|
| 5695 |
period_spec = cx.REPORT_TYPES[table]
|
| 5696 |
windows = (([(task_start, task_end)] if task_end is not None else []) if first_load
|
|
|
|
| 5888 |
"rows": total_rows, "columns": column_count, "provenance": prov,
|
| 5889 |
"report": rtype, "window": plan["cause"], "incremental": plan["incremental"],
|
| 5890 |
"operations": operations}
|
| 5891 |
+
if resync_count:
|
| 5892 |
+
report["tables"][table]["resync"] = {"landed": resync_count, "left": resync_left}
|
| 5893 |
_record_state(con, key, "done", total_rows)
|
| 5894 |
_record_state(con, entity_key("sync_tick_" + table), "done", total_rows)
|
| 5895 |
if not any(v["state"] in ("refused", "throttled") for v in operations.values()):
|
|
|
|
| 8007 |
and queued_resync(con, "long_term_storage_fees") is None
|
| 8008 |
and con.execute(f"SELECT _period_start, _period_end, count(*) FROM {fees} "
|
| 8009 |
"GROUP BY 1, 2 ORDER BY 1").fetchall() == stored)
|
| 8010 |
+
# S79-T23: a scheduled pass inside the daily floor drains every queued window in one turn.
|
| 8011 |
+
queue = [list(window[:2]) for window in stored]
|
| 8012 |
+
_record_state(con, entity_key(RESYNC_PREFIX + "long_term_storage_fees"), "pending", 1,
|
| 8013 |
+
json.dumps({"windows": queue}))
|
| 8014 |
+
_record_state(con, entity_key("sync_tick_long_term_storage_fees"), "done")
|
| 8015 |
+
clock[0] += timedelta(seconds=60)
|
| 8016 |
+
before = len(calls)
|
| 8017 |
+
held = _sync_candidate(rt.key, tables=("long_term_storage_fees",), rt=rt, log=lambda *_: None,
|
| 8018 |
+
scheduled=True)["tables"]["long_term_storage_fees"]
|
| 8019 |
+
ck("NC: a queued resync still waits out Amazon's own report floor",
|
| 8020 |
+
held.get("state") == "deferred" and len(calls) == before
|
| 8021 |
+
and len(queued_resyncs(con, "long_term_storage_fees")) == len(queue))
|
| 8022 |
+
clock[0] += timedelta(seconds=cx.REPORT_TYPES["long_term_storage_fees"]["min_interval_s"])
|
| 8023 |
+
lines, before = [], len(calls)
|
| 8024 |
+
drained = _sync_candidate(rt.key, tables=("long_term_storage_fees",), rt=rt, log=lines.append,
|
| 8025 |
+
scheduled=True)["tables"]["long_term_storage_fees"]
|
| 8026 |
+
ck("a scheduled resync turn drains several windows inside the daily floor, one log line",
|
| 8027 |
+
len(queue) > 1 and len(calls) - before == len(queue)
|
| 8028 |
+
and drained.get("resync") == {"landed": len(queue), "left": 0}
|
| 8029 |
+
and f"Amazon resync long_term_storage_fees: {len(queue)} windows landed, 0 left" in lines
|
| 8030 |
+
and queued_resync(con, "long_term_storage_fees") is None
|
| 8031 |
+
and read_watermark(con, "long_term_storage_fees") == mark)
|
| 8032 |
+
clock[0] += timedelta(seconds=cx.REPORT_TYPES["long_term_storage_fees"]["min_interval_s"] + 60)
|
| 8033 |
+
idle = _sync_candidate(rt.key, tables=("long_term_storage_fees",), rt=rt, log=lambda *_: None,
|
| 8034 |
+
scheduled=True)["tables"]["long_term_storage_fees"]
|
| 8035 |
+
ck("NC: with the queue empty the daily floor defers the table again",
|
| 8036 |
+
idle.get("state") == "deferred" and "once a day" in idle.get("reason", ""))
|
| 8037 |
finally:
|
| 8038 |
con.close()
|
| 8039 |
|
platform/harness/datastore.py
CHANGED
|
@@ -394,7 +394,9 @@ ENTITIES = {
|
|
| 394 |
# NOTE: no `active` field on account.account in this Odoo version (learned 2026-07-11 —
|
| 395 |
# a ('active','in',...) domain 500s); treat as non-archivable.
|
| 396 |
"model": "account.account", "archivable": False,
|
| 397 |
-
|
|
|
|
|
|
|
| 398 |
},
|
| 399 |
# ⭐⭐ W37-T13 (owner: stock moved IN and OUT per SKU) — MEASURED FIRST in
|
| 400 |
# `proto/P1-stock-moves.md`, and every clause below is a finding rather than a guess.
|
|
@@ -853,7 +855,7 @@ BATCH = 2000
|
|
| 853 |
# (see [[invoice-line-agent-commission]]): every row would read "not an agent" indistinguishably
|
| 854 |
# from "unknown".
|
| 855 |
BOOL_FIELDS = ("active", "agent", "salesman_as_agent", "settled", "commission_free",
|
| 856 |
-
"is_internal_transfer")
|
| 857 |
|
| 858 |
# ⭐ WAVE 29 — fields that are genuinely INTEGER COUNTS, and this list exists because its absence
|
| 859 |
# shipped a silent defect within minutes of `customer_rank` being added to the res_partner spec.
|
|
@@ -3255,7 +3257,7 @@ def _already(con, key, rec):
|
|
| 3255 |
return bool(con.execute(f"SELECT 1 FROM {key} WHERE id=?", [rec["id"]]).fetchone())
|
| 3256 |
|
| 3257 |
|
| 3258 |
-
def sync_all(entities=None, max_batches=200, log=print):
|
| 3259 |
selected = list(ENTITIES) if entities is None else list(entities)
|
| 3260 |
if entities is not None and not selected:
|
| 3261 |
return {}
|
|
@@ -3279,6 +3281,12 @@ def sync_all(entities=None, max_batches=200, log=print):
|
|
| 3279 |
t0 = time.time()
|
| 3280 |
phase, pulled = sync_entity(key, max_batches=max_batches, log=log)
|
| 3281 |
out[key] = {"phase": phase, "pulled": pulled, "secs": round(time.time() - t0, 1)}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3282 |
if rt is not None and out:
|
| 3283 |
persist_odoo_mirror(rt.key, rt, out, log=log)
|
| 3284 |
# A full pass that lands every entity in live mode latches process readiness in-process — so
|
|
|
|
| 394 |
# NOTE: no `active` field on account.account in this Odoo version (learned 2026-07-11 —
|
| 395 |
# a ('active','in',...) domain 500s); treat as non-archivable.
|
| 396 |
"model": "account.account", "archivable": False,
|
| 397 |
+
# non_trade feeds the line-grain Overdue (ar.AGED_LINE_DOMAIN). Not a grid column, so unaudited.
|
| 398 |
+
"fields": ["code", "name", "account_type", "non_trade", "write_date"],
|
| 399 |
+
"backfill_fields": ["non_trade"],
|
| 400 |
},
|
| 401 |
# ⭐⭐ W37-T13 (owner: stock moved IN and OUT per SKU) — MEASURED FIRST in
|
| 402 |
# `proto/P1-stock-moves.md`, and every clause below is a finding rather than a guess.
|
|
|
|
| 855 |
# (see [[invoice-line-agent-commission]]): every row would read "not an agent" indistinguishably
|
| 856 |
# from "unknown".
|
| 857 |
BOOL_FIELDS = ("active", "agent", "salesman_as_agent", "settled", "commission_free",
|
| 858 |
+
"is_internal_transfer", "non_trade")
|
| 859 |
|
| 860 |
# ⭐ WAVE 29 — fields that are genuinely INTEGER COUNTS, and this list exists because its absence
|
| 861 |
# shipped a silent defect within minutes of `customer_rank` being added to the res_partner spec.
|
|
|
|
| 3257 |
return bool(con.execute(f"SELECT 1 FROM {key} WHERE id=?", [rec["id"]]).fetchone())
|
| 3258 |
|
| 3259 |
|
| 3260 |
+
def sync_all(entities=None, max_batches=200, log=print, after_entity=None):
|
| 3261 |
selected = list(ENTITIES) if entities is None else list(entities)
|
| 3262 |
if entities is not None and not selected:
|
| 3263 |
return {}
|
|
|
|
| 3281 |
t0 = time.time()
|
| 3282 |
phase, pulled = sync_entity(key, max_batches=max_batches, log=log)
|
| 3283 |
out[key] = {"phase": phase, "pulled": pulled, "secs": round(time.time() - t0, 1)}
|
| 3284 |
+
if after_entity is not None:
|
| 3285 |
+
# Beside this entity's copy, before the next pull. A hook failure never fails the pass.
|
| 3286 |
+
try:
|
| 3287 |
+
after_entity(key)
|
| 3288 |
+
except Exception as exc: # noqa: BLE001
|
| 3289 |
+
log(f" {key}: after-entity hook failed ({type(exc).__name__})")
|
| 3290 |
if rt is not None and out:
|
| 3291 |
persist_odoo_mirror(rt.key, rt, out, log=log)
|
| 3292 |
# A full pass that lands every entity in live mode latches process readiness in-process — so
|
platform/harness/semantic.py
CHANGED
|
@@ -134,6 +134,20 @@ class ModelError(Exception):
|
|
| 134 |
"""A model-file problem (unknown key, bad reference, unparseable expr)."""
|
| 135 |
|
| 136 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 137 |
class UnknownColumnError(ModelError):
|
| 138 |
"""A filter/group request named a column the topic does not declare."""
|
| 139 |
|
|
@@ -1019,11 +1033,10 @@ def _v_ar_overdue(date_from, date_to, team_id):
|
|
| 1019 |
as_of = con.execute("SELECT CURRENT_DATE").fetchone()[0]
|
| 1020 |
finally:
|
| 1021 |
con.close()
|
| 1022 |
-
|
| 1023 |
-
# Only a line-grain binding can tie: a document sum leaves out unapplied payments, so the
|
| 1024 |
-
# gap stays red until the metric reads account_move_line (Rule 8).
|
| 1025 |
res = store_query(_model()["metrics"]["ar_overdue"]["topic"], ["ar_overdue"])
|
| 1026 |
ours = res["rows"][0]["ar_overdue"] if res["rows"] else 0
|
|
|
|
| 1027 |
return ours, tie["b"]
|
| 1028 |
|
| 1029 |
|
|
@@ -1191,7 +1204,11 @@ def validate_metric(key, date_from=None, date_to=None, team_id=None, tol=0.01, *
|
|
| 1191 |
if key == 'amazon_order_count':
|
| 1192 |
tol = 0 # Order counts reconcile exactly; a money tolerance cannot forgive missing orders.
|
| 1193 |
else:
|
| 1194 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1195 |
if m.get('date_age'):
|
| 1196 |
gap = 0 if ours == independent else 1
|
| 1197 |
elif m.get('format') == 'date':
|
|
@@ -1231,6 +1248,21 @@ def _store_con():
|
|
| 1231 |
return DS.ro_cursor() # cursor on the shared instance — no sync-writer contention
|
| 1232 |
|
| 1233 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1234 |
def _store_unready_cause(DS):
|
| 1235 |
"""D-513: the condition `ready()` actually hit. The old sentence always blamed a warm-up after
|
| 1236 |
a restart, which is rarely the cause (another process holding the file is the usual one) and
|
|
@@ -1769,6 +1801,12 @@ def store_query(topic, measures, group_by=None, grain=None, date_from=None, date
|
|
| 1769 |
raise ModelError(f'Standard metric source {source!r} needs its declared field backfill')
|
| 1770 |
if metric_key == 'shipped':
|
| 1771 |
_check_shipped_links(date_from, date_to)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1772 |
foreign_vals = {}
|
| 1773 |
for k in foreign:
|
| 1774 |
sub = store_query(_model()["metrics"][k]["topic"], [k], date_from=date_from,
|
|
|
|
| 134 |
"""A model-file problem (unknown key, bad reference, unparseable expr)."""
|
| 135 |
|
| 136 |
|
| 137 |
+
class NotSyncedError(ModelError):
|
| 138 |
+
"""A metric reads mirror fields whose declared Odoo backfill has not finished."""
|
| 139 |
+
|
| 140 |
+
code = "not_synced"
|
| 141 |
+
|
| 142 |
+
def __init__(self, metric, sources):
|
| 143 |
+
self.metric = str(metric.get("key") or "")
|
| 144 |
+
self.sources = list(sources)
|
| 145 |
+
super().__init__(
|
| 146 |
+
f"{metric.get('label') or self.metric} is not available yet. The Odoo fields it reads "
|
| 147 |
+
f"have not finished syncing into this data cache ({', '.join(self.sources)}). "
|
| 148 |
+
"Run Sync now on the Odoo connector, or ask again after the next scheduled sync.")
|
| 149 |
+
|
| 150 |
+
|
| 151 |
class UnknownColumnError(ModelError):
|
| 152 |
"""A filter/group request named a column the topic does not declare."""
|
| 153 |
|
|
|
|
| 1033 |
as_of = con.execute("SELECT CURRENT_DATE").fetchone()[0]
|
| 1034 |
finally:
|
| 1035 |
con.close()
|
| 1036 |
+
# The store side first: a mirror without the line residual refuses before any Odoo read.
|
|
|
|
|
|
|
| 1037 |
res = store_query(_model()["metrics"]["ar_overdue"]["topic"], ["ar_overdue"])
|
| 1038 |
ours = res["rows"][0]["ar_overdue"] if res["rows"] else 0
|
| 1039 |
+
tie = ar.overdue_tie(ar.aged_receivable(as_of))
|
| 1040 |
return ours, tie["b"]
|
| 1041 |
|
| 1042 |
|
|
|
|
| 1204 |
if key == 'amazon_order_count':
|
| 1205 |
tol = 0 # Order counts reconcile exactly; a money tolerance cannot forgive missing orders.
|
| 1206 |
else:
|
| 1207 |
+
try:
|
| 1208 |
+
ours, independent = fn(date_from, date_to, team_id)
|
| 1209 |
+
except NotSyncedError as exc:
|
| 1210 |
+
return {"check": f"semantic.{key}: mirror fields not synced", "ok": False, "gap": None,
|
| 1211 |
+
"state": exc.code, "reason": str(exc), "sources": exc.sources}
|
| 1212 |
if m.get('date_age'):
|
| 1213 |
gap = 0 if ours == independent else 1
|
| 1214 |
elif m.get('format') == 'date':
|
|
|
|
| 1248 |
return DS.ro_cursor() # cursor on the shared instance — no sync-writer contention
|
| 1249 |
|
| 1250 |
|
| 1251 |
+
def _pending_backfills(sources, cur=None):
|
| 1252 |
+
"""The listed mirror entities whose declared column backfill has not finished."""
|
| 1253 |
+
if not sources:
|
| 1254 |
+
return []
|
| 1255 |
+
from harness import datastore
|
| 1256 |
+
con = cur if cur is not None else _store_con()
|
| 1257 |
+
try:
|
| 1258 |
+
# A column ALTERed in after the last pass stays absent or NULL until its marker reads done.
|
| 1259 |
+
return [s for s in sources
|
| 1260 |
+
if not datastore._columns_backfilled(con, s, datastore.ENTITIES[s])]
|
| 1261 |
+
finally:
|
| 1262 |
+
if cur is None:
|
| 1263 |
+
con.close()
|
| 1264 |
+
|
| 1265 |
+
|
| 1266 |
def _store_unready_cause(DS):
|
| 1267 |
"""D-513: the condition `ready()` actually hit. The old sentence always blamed a warm-up after
|
| 1268 |
a restart, which is rarely the cause (another process holding the file is the usual one) and
|
|
|
|
| 1801 |
raise ModelError(f'Standard metric source {source!r} needs its declared field backfill')
|
| 1802 |
if metric_key == 'shipped':
|
| 1803 |
_check_shipped_links(date_from, date_to)
|
| 1804 |
+
for metric_key in base_keys:
|
| 1805 |
+
if metric_key in _STANDARD_METRICS:
|
| 1806 |
+
continue
|
| 1807 |
+
pending = _pending_backfills(_model()['metrics'][metric_key].get('required_backfills'), cur=cur)
|
| 1808 |
+
if pending:
|
| 1809 |
+
raise NotSyncedError(_model()['metrics'][metric_key], pending)
|
| 1810 |
foreign_vals = {}
|
| 1811 |
for k in foreign:
|
| 1812 |
sub = store_query(_model()["metrics"][k]["topic"], [k], date_from=date_from,
|
platform/model/metrics/receivables.yml
CHANGED
|
@@ -1,5 +1,5 @@
|
|
| 1 |
# Metrics: receivables (topic receivables) — AR exposure, matching the Collections conventions
|
| 2 |
-
# (amount_residual_signed nets credit notes
|
| 3 |
topic: receivables
|
| 4 |
metrics:
|
| 5 |
- key: customer_open_documents
|
|
@@ -41,17 +41,22 @@ metrics:
|
|
| 41 |
method: ar_outstanding
|
| 42 |
note: "Store-path outstanding must equal the live Odoo aggregate over the same domain, to the cent."
|
| 43 |
|
| 44 |
-
# R37 / C10:
|
| 45 |
-
#
|
| 46 |
-
#
|
| 47 |
- key: ar_overdue
|
|
|
|
| 48 |
label: Overdue
|
| 49 |
agg: sum
|
| 50 |
-
field:
|
| 51 |
-
store_filter_sql: "CAST(
|
|
|
|
| 52 |
format: usd
|
| 53 |
description: "Overdue, defined as Odoo Aged Receivable overdue: open receivable past its due date with no grace, New Customer Hold due at once, credits and unapplied payments netted per customer."
|
| 54 |
-
ai_context: "
|
|
|
|
|
|
|
|
|
|
| 55 |
|
| 56 |
- key: open_invoices
|
| 57 |
label: Open invoices
|
|
|
|
| 1 |
# Metrics: receivables (topic receivables) — AR exposure, matching the Collections conventions
|
| 2 |
+
# (amount_residual_signed nets credit notes). ar_overdue reads receivable lines (topic gl_lines).
|
| 3 |
topic: receivables
|
| 4 |
metrics:
|
| 5 |
- key: customer_open_documents
|
|
|
|
| 41 |
method: ar_outstanding
|
| 42 |
note: "Store-path outstanding must equal the live Odoo aggregate over the same domain, to the cent."
|
| 43 |
|
| 44 |
+
# R37 / C10: Overdue at LINE grain, the Odoo Aged Receivable basis `_v_ar_overdue` ties to
|
| 45 |
+
# `ar.overdue_tie` (AGED_LINE_DOMAIN plus the past-due clause). Line grain keeps unapplied payments.
|
| 46 |
+
# A mirror whose line residual or account non_trade backfill is unfinished refuses (not_synced).
|
| 47 |
- key: ar_overdue
|
| 48 |
+
topic: gl_lines
|
| 49 |
label: Overdue
|
| 50 |
agg: sum
|
| 51 |
+
field: amount_residual
|
| 52 |
+
store_filter_sql: "a.account_type = 'asset_receivable' AND NOT COALESCE(CAST(a.non_trade AS BOOLEAN), FALSE) AND l.parent_state = 'posted' AND NOT COALESCE(CAST(l.reconciled AS BOOLEAN), FALSE) AND CAST(l.date AS DATE) <= CURRENT_DATE AND (CAST(l.date_maturity AS DATE) < CURRENT_DATE OR (l.date_maturity IS NULL AND CAST(l.date AS DATE) < CURRENT_DATE))"
|
| 53 |
+
required_backfills: [account_move_line, account_account]
|
| 54 |
format: usd
|
| 55 |
description: "Overdue, defined as Odoo Aged Receivable overdue: open receivable past its due date with no grace, New Customer Hold due at once, credits and unapplied payments netted per customer."
|
| 56 |
+
ai_context: "Sums the residual of open trade receivable journal lines past their due date, so credits and unapplied payments net in. Matches the customer database field ar_overdue (label Overdue). The 5-day grace number is the field ar_overdue_past_grace (label Overdue past grace)."
|
| 57 |
+
validate:
|
| 58 |
+
method: ar_overdue
|
| 59 |
+
note: "Store-path line residual past due must equal Odoo account.move.line amount_residual over the Aged Receivable domain, to the cent."
|
| 60 |
|
| 61 |
- key: open_invoices
|
| 62 |
label: Open invoices
|
platform/model/skills/odoo/overdue-receivables.md
CHANGED
|
@@ -24,14 +24,16 @@ Wait for a completed source snapshot. Record its timestamp and the query window
|
|
| 24 |
Do not store the answer's figures in this playbook.
|
| 25 |
|
| 26 |
## Query
|
| 27 |
-
Use the registered ar_overdue metric. Resolve $today to the date used by that metric.
|
|
|
|
| 28 |
The front-matter query is a parameterized read request, never an instruction to change a database.
|
| 29 |
|
| 30 |
## Traps
|
| 31 |
-
A due date equal to today is not overdue.
|
| 32 |
|
| 33 |
## Check the answer
|
| 34 |
-
|
|
|
|
| 35 |
Use read-only source aggregates through the workspace's authorized source reader.
|
| 36 |
Keep the query, resolved domain, snapshot time and comparison in the answer's evidence.
|
| 37 |
An unavailable source check leaves the result unverified. A mirror checking itself is insufficient.
|
|
|
|
| 24 |
Do not store the answer's figures in this playbook.
|
| 25 |
|
| 26 |
## Query
|
| 27 |
+
Use the registered ar_overdue metric. Resolve $today to the date used by that metric. It sums the residual of open trade receivable journal lines due strictly before $today, so credits and unapplied payments net in. A not_synced answer means its mirror fields are still syncing, so the metric is unavailable.
|
| 28 |
+
The front-matter query reads invoice documents only. Use it only when the metric is unavailable, and say it leaves out unapplied payments.
|
| 29 |
The front-matter query is a parameterized read request, never an instruction to change a database.
|
| 30 |
|
| 31 |
## Traps
|
| 32 |
+
A due date equal to today is not overdue. A line with no due date is due on its entry date. Credits and unapplied payments reduce the balance. Today's residuals cannot reconstruct a past aging report.
|
| 33 |
|
| 34 |
## Check the answer
|
| 35 |
+
For the metric, read-group account.move.line amount_residual:sum over posted, unreconciled trade receivable lines dated on or before $today and due before it. Require a zero rounded currency delta. Report the cutoff timezone and timestamp.
|
| 36 |
+
The front-matter source_check ties the document fallback only.
|
| 37 |
Use read-only source aggregates through the workspace's authorized source reader.
|
| 38 |
Keep the query, resolved domain, snapshot time and comparison in the answer's evidence.
|
| 39 |
An unavailable source check leaves the result unverified. A mirror checking itself is insufficient.
|
platform/model/topics/gl_lines.yml
CHANGED
|
@@ -38,3 +38,5 @@ ai_context: >
|
|
| 38 |
negates them to read positively. This topic is company-level: business-unit filters do not
|
| 39 |
apply here (use sales_lines for BU questions). The newest month is often still being booked —
|
| 40 |
a sharply lower final month usually means an OPEN period, not a cost win (close-lag).
|
|
|
|
|
|
|
|
|
| 38 |
negates them to read positively. This topic is company-level: business-unit filters do not
|
| 39 |
apply here (use sales_lines for BU questions). The newest month is often still being booked —
|
| 40 |
a sharply lower final month usually means an OPEN period, not a cost win (close-lag).
|
| 41 |
+
It also carries ar_overdue (Overdue): open trade receivable lines past their due date, the
|
| 42 |
+
Odoo Aged Receivable basis, as an as-of-today balance that takes no date window.
|
platform/modules/competitor_watch.py
CHANGED
|
@@ -31,6 +31,9 @@ HOLDING = ("Open", "Watching", "Ignore")
|
|
| 31 |
DEFAULTS = {"undercut_pct": 5.0, "rank_surge_pct": 50.0}
|
| 32 |
DUE_DAYS = 7
|
| 33 |
NOT_READY = "Comparison starts after the next weekly pull."
|
|
|
|
|
|
|
|
|
|
| 34 |
LIMIT = ("Each competitor watch document is limited to 128 MB by the store. About 200,000 alerts "
|
| 35 |
"fit in one. Split the alerts document by year when it nears that size.")
|
| 36 |
|
|
@@ -80,9 +83,15 @@ def _mirror_path(tenant_key, st):
|
|
| 80 |
return amazon_store._tenant_mirror_path(tenant_key, rt)
|
| 81 |
|
| 82 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 83 |
@contextlib.contextmanager
|
| 84 |
def _reader(tenant_key, st):
|
| 85 |
-
"""A read-only cursor on the tenant mirror,
|
|
|
|
|
|
|
| 86 |
path = Path(_mirror_path(tenant_key, st))
|
| 87 |
if not path.exists():
|
| 88 |
yield None
|
|
@@ -93,27 +102,27 @@ def _reader(tenant_key, st):
|
|
| 93 |
except Exception: # noqa: BLE001
|
| 94 |
shared = False
|
| 95 |
# The process holds one writer on its own mirror: a separate read-only connect would refuse.
|
| 96 |
-
if shared:
|
| 97 |
-
cur = datastore.ro_cursor()
|
| 98 |
-
try:
|
| 99 |
-
yield cur
|
| 100 |
-
finally:
|
| 101 |
-
cur.close()
|
| 102 |
-
return
|
| 103 |
-
import duckdb
|
| 104 |
-
con = duckdb.connect(str(path), read_only=True)
|
| 105 |
try:
|
| 106 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 107 |
finally:
|
| 108 |
-
|
| 109 |
|
| 110 |
|
| 111 |
def _columns(cur, table):
|
|
|
|
| 112 |
try:
|
| 113 |
return {r[0] for r in cur.execute(
|
| 114 |
"SELECT column_name FROM information_schema.columns WHERE table_name = ?", [table]).fetchall()}
|
| 115 |
-
except Exception:
|
| 116 |
-
|
| 117 |
|
| 118 |
|
| 119 |
def _sqp(cur):
|
|
@@ -415,7 +424,8 @@ def on_amazon_sync(tenant_key, synced_tables):
|
|
| 415 |
try:
|
| 416 |
def _note(doc):
|
| 417 |
doc = doc if isinstance(doc, dict) else {}
|
| 418 |
-
doc["last_error"] =
|
|
|
|
| 419 |
return doc
|
| 420 |
_rt(tenant_key).update(KEY_WEEKS, _note)
|
| 421 |
except Exception: # noqa: BLE001
|
|
|
|
| 31 |
DEFAULTS = {"undercut_pct": 5.0, "rank_surge_pct": 50.0}
|
| 32 |
DUE_DAYS = 7
|
| 33 |
NOT_READY = "Comparison starts after the next weekly pull."
|
| 34 |
+
#: The mirror holds the tables but this read failed or was busy. `{cause}` is the error class.
|
| 35 |
+
READ_FAILED = ("The Amazon mirror could not be read just now ({cause}), so no search term was compared. "
|
| 36 |
+
"The next Amazon sync reads it again.")
|
| 37 |
LIMIT = ("Each competitor watch document is limited to 128 MB by the store. About 200,000 alerts "
|
| 38 |
"fit in one. Split the alerts document by year when it nears that size.")
|
| 39 |
|
|
|
|
| 83 |
return amazon_store._tenant_mirror_path(tenant_key, rt)
|
| 84 |
|
| 85 |
|
| 86 |
+
class _ReadFailed(Exception):
|
| 87 |
+
"""A mirror read failed, was locked or busy. Never read as an absent table."""
|
| 88 |
+
|
| 89 |
+
|
| 90 |
@contextlib.contextmanager
|
| 91 |
def _reader(tenant_key, st):
|
| 92 |
+
"""A read-only cursor on the tenant mirror, None when the mirror file does not exist.
|
| 93 |
+
|
| 94 |
+
An open failure raises `_ReadFailed` naming its error class."""
|
| 95 |
path = Path(_mirror_path(tenant_key, st))
|
| 96 |
if not path.exists():
|
| 97 |
yield None
|
|
|
|
| 102 |
except Exception: # noqa: BLE001
|
| 103 |
shared = False
|
| 104 |
# The process holds one writer on its own mirror: a separate read-only connect would refuse.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 105 |
try:
|
| 106 |
+
if shared:
|
| 107 |
+
cur = datastore.ro_cursor()
|
| 108 |
+
else:
|
| 109 |
+
import duckdb
|
| 110 |
+
cur = duckdb.connect(str(path), read_only=True)
|
| 111 |
+
except Exception as exc: # noqa: BLE001
|
| 112 |
+
raise _ReadFailed(type(exc).__name__) from exc
|
| 113 |
+
try:
|
| 114 |
+
yield cur
|
| 115 |
finally:
|
| 116 |
+
cur.close()
|
| 117 |
|
| 118 |
|
| 119 |
def _columns(cur, table):
|
| 120 |
+
"""The table's column names, an empty set when it is absent. A failed read raises `_ReadFailed`."""
|
| 121 |
try:
|
| 122 |
return {r[0] for r in cur.execute(
|
| 123 |
"SELECT column_name FROM information_schema.columns WHERE table_name = ?", [table]).fetchall()}
|
| 124 |
+
except Exception as exc: # noqa: BLE001
|
| 125 |
+
raise _ReadFailed(type(exc).__name__) from exc
|
| 126 |
|
| 127 |
|
| 128 |
def _sqp(cur):
|
|
|
|
| 424 |
try:
|
| 425 |
def _note(doc):
|
| 426 |
doc = doc if isinstance(doc, dict) else {}
|
| 427 |
+
doc["last_error"] = (READ_FAILED.format(cause=exc) if isinstance(exc, _ReadFailed) else
|
| 428 |
+
f"The weekly comparison failed ({type(exc).__name__}): {exc}")
|
| 429 |
return doc
|
| 430 |
_rt(tenant_key).update(KEY_WEEKS, _note)
|
| 431 |
except Exception: # noqa: BLE001
|
platform/modules/opportunity_finder.py
CHANGED
|
@@ -22,6 +22,7 @@ import contextlib
|
|
| 22 |
import contextvars
|
| 23 |
import datetime as dt
|
| 24 |
import hashlib
|
|
|
|
| 25 |
import os
|
| 26 |
import threading
|
| 27 |
from pathlib import Path
|
|
@@ -68,12 +69,16 @@ LIMIT = ("Each week of history is one store document, limited to 128 MB. About 3
|
|
| 68 |
|
| 69 |
#: A mirror probe longer than this is interrupted and reads as busy, under the 45 s request deadline.
|
| 70 |
PROBE_SECONDS = 20.0
|
|
|
|
|
|
|
| 71 |
#: Mirror facts and rising lists kept per process, keyed on the mirror mark. Oldest goes first.
|
| 72 |
MEMO_KEEP = 16
|
| 73 |
|
| 74 |
#: Accounts that sign in for a process, never a person who can own an item.
|
| 75 |
SERVICE_ACCOUNTS = ("admin", "automation", "scheduler", "qa-runner", "leadership")
|
| 76 |
|
|
|
|
|
|
|
| 77 |
|
| 78 |
class UnknownItem(KeyError, ValueError):
|
| 79 |
"""An item id the finder does not hold. A KeyError for the router, a ValueError for C4."""
|
|
@@ -145,14 +150,16 @@ _MEMO = {}
|
|
| 145 |
_MEMO_LOCK = threading.Lock()
|
| 146 |
|
| 147 |
|
| 148 |
-
def _remember(key, build):
|
| 149 |
-
"""`build()` once per key. A failed build is not kept
|
| 150 |
if key is None:
|
| 151 |
return build()
|
| 152 |
with _MEMO_LOCK:
|
| 153 |
if key in _MEMO:
|
| 154 |
return _MEMO[key]
|
| 155 |
value = build()
|
|
|
|
|
|
|
| 156 |
with _MEMO_LOCK:
|
| 157 |
_MEMO[key] = value
|
| 158 |
while len(_MEMO) > MEMO_KEEP:
|
|
@@ -160,13 +167,18 @@ def _remember(key, build):
|
|
| 160 |
return value
|
| 161 |
|
| 162 |
|
| 163 |
-
def _mark(
|
| 164 |
-
"""Moves whenever the mirror rows can have changed: an install, a
|
|
|
|
|
|
|
|
|
|
| 165 |
from harness import datastore
|
|
|
|
|
|
|
| 166 |
generation = None
|
| 167 |
if shared:
|
| 168 |
with contextlib.suppress(Exception):
|
| 169 |
-
generation = datastore.mirror_generation()
|
| 170 |
stamps = []
|
| 171 |
for part in (Path(path), Path(str(path) + ".wal")):
|
| 172 |
try:
|
|
@@ -178,36 +190,45 @@ def _mark(path, shared):
|
|
| 178 |
|
| 179 |
|
| 180 |
@contextlib.contextmanager
|
| 181 |
-
def _reader(tenant_key, st):
|
| 182 |
"""A read cursor on the tenant mirror, None when there is no mirror, `_Unreadable` when it cannot open.
|
| 183 |
|
| 184 |
-
|
| 185 |
-
`datastore.ro_cursor()` on the serving
|
|
|
|
| 186 |
from harness import datastore
|
| 187 |
path = Path(_mirror_path(tenant_key, st))
|
| 188 |
try:
|
| 189 |
shared = Path(datastore.DB_PATH).resolve() == path.resolve()
|
| 190 |
except Exception: # noqa: BLE001
|
| 191 |
shared = False
|
| 192 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 193 |
yield None
|
| 194 |
return
|
| 195 |
cur = None
|
| 196 |
try:
|
| 197 |
-
if
|
|
|
|
|
|
|
| 198 |
if hasattr(st, "assert_datastore_matches"):
|
| 199 |
st.assert_datastore_matches()
|
| 200 |
cur = datastore.ro_cursor()
|
| 201 |
else:
|
| 202 |
import duckdb
|
| 203 |
-
cur = duckdb.connect(str(
|
|
|
|
| 204 |
except Exception as exc: # noqa: BLE001
|
| 205 |
if cur is not None:
|
| 206 |
with contextlib.suppress(Exception):
|
| 207 |
cur.close()
|
| 208 |
-
yield _Unreadable(type(exc).__name__)
|
| 209 |
return
|
| 210 |
-
token = _MARK.set(
|
| 211 |
try:
|
| 212 |
yield cur
|
| 213 |
finally:
|
|
@@ -221,12 +242,22 @@ def _interrupt(cur):
|
|
| 221 |
|
| 222 |
|
| 223 |
def _bounded(cur, sql, params=()):
|
| 224 |
-
"""Every row of one
|
| 225 |
|
| 226 |
-
|
| 227 |
if cur is None or isinstance(cur, _Unreadable):
|
| 228 |
raise _ReadFailed(getattr(cur, "cause", "NoMirror"))
|
| 229 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 230 |
timer.daemon = True
|
| 231 |
timer.start()
|
| 232 |
try:
|
|
@@ -234,6 +265,8 @@ def _bounded(cur, sql, params=()):
|
|
| 234 |
except Exception as exc: # noqa: BLE001
|
| 235 |
raise _ReadFailed(type(exc).__name__) from exc
|
| 236 |
finally:
|
|
|
|
|
|
|
| 237 |
timer.cancel()
|
| 238 |
|
| 239 |
|
|
@@ -241,19 +274,13 @@ def _types(cur, table):
|
|
| 241 |
"""{column: data type} of a mirror table, {} when the table is absent. A failed read raises."""
|
| 242 |
if not table.replace("_", "").isalnum():
|
| 243 |
raise ValueError(f"{table!r} is not a table name.")
|
| 244 |
-
# A literal, not a parameter: DuckDB ignores interrupt() on a parameterized execute.
|
| 245 |
return {str(n): str(t) for n, t in _bounded(
|
| 246 |
cur, f"SELECT column_name, data_type FROM information_schema.columns WHERE table_name = '{table}'")}
|
| 247 |
|
| 248 |
|
| 249 |
def _columns(cur, table):
|
| 250 |
-
|
| 251 |
-
|
| 252 |
-
try:
|
| 253 |
-
return {r[0] for r in cur.execute(
|
| 254 |
-
"SELECT column_name FROM information_schema.columns WHERE table_name = ?", [table]).fetchall()}
|
| 255 |
-
except Exception: # noqa: BLE001
|
| 256 |
-
return set()
|
| 257 |
|
| 258 |
|
| 259 |
def iso_period(value):
|
|
@@ -269,11 +296,11 @@ def _our_asins(cur):
|
|
| 269 |
"""SQP ASINs plus listing `asin1`: the ASINs the seller owns."""
|
| 270 |
ours = set()
|
| 271 |
if "asin" in _columns(cur, SQP_TABLE):
|
| 272 |
-
ours |= {_text(r[0]) for r in
|
| 273 |
-
f'SELECT DISTINCT CAST("asin" AS VARCHAR) FROM {SQP_TABLE} WHERE "asin" IS NOT NULL')
|
| 274 |
if "asin1" in _columns(cur, LISTINGS_TABLE):
|
| 275 |
-
ours |= {_text(r[0]) for r in
|
| 276 |
-
f'SELECT DISTINCT CAST("asin1" AS VARCHAR) FROM {LISTINGS_TABLE} WHERE "asin1" IS NOT NULL')
|
| 277 |
return {a for a in ours if a}
|
| 278 |
|
| 279 |
|
|
@@ -297,18 +324,25 @@ def _derived_departments(cur, ours):
|
|
| 297 |
"""R49 default: every department where a brand analytics row clicked one of our ASINs, any period."""
|
| 298 |
if not ours or "clicked_asin" not in _columns(cur, BA_TABLE):
|
| 299 |
return []
|
| 300 |
-
rows =
|
| 301 |
-
f'SELECT DISTINCT {_DEPT} FROM {BA_TABLE} '
|
| 302 |
f'WHERE trim(CAST("clicked_asin" AS VARCHAR)) IN (SELECT unnest(?::VARCHAR[])) '
|
| 303 |
-
f'AND "department_name" IS NOT NULL', [sorted(ours)])
|
| 304 |
return sorted(_text(r[0]) for r in rows if _text(r[0]))
|
| 305 |
|
| 306 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 307 |
def _facts(cur, fresh=False):
|
| 308 |
-
"""The mirror facts every read starts from, once per mirror mark: columns, newest period, ours, derived.
|
|
|
|
|
|
|
| 309 |
def build():
|
| 310 |
types = _types(cur, BA_TABLE)
|
| 311 |
-
facts = {"columns": set(types), "period": None, "ours": set(), "derived": []
|
|
|
|
| 312 |
if not set(BA_NEED) <= facts["columns"]:
|
| 313 |
return facts
|
| 314 |
facts["period"] = _period(cur, types.get("_period_start", "VARCHAR"))
|
|
@@ -317,16 +351,22 @@ def _facts(cur, fresh=False):
|
|
| 317 |
try:
|
| 318 |
facts["ours"] = _our_asins(cur)
|
| 319 |
facts["derived"] = _derived_departments(cur, facts["ours"])
|
|
|
|
|
|
|
| 320 |
except Exception as exc: # noqa: BLE001
|
| 321 |
raise _ReadFailed(type(exc).__name__) from exc
|
| 322 |
return facts
|
| 323 |
mark = _MARK.get()
|
| 324 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 325 |
|
| 326 |
|
| 327 |
def _source(cur, st, fresh=False):
|
| 328 |
"""What the mirror holds now: {period, departments, ours, problems}. Departments follow the setting."""
|
| 329 |
-
out = {"period": None, "departments": [], "ours": set(), "problems": [], "derived": []}
|
| 330 |
if cur is None:
|
| 331 |
out["problems"].append(NO_MIRROR)
|
| 332 |
return out
|
|
@@ -335,7 +375,8 @@ def _source(cur, st, fresh=False):
|
|
| 335 |
except _ReadFailed as exc:
|
| 336 |
out["problems"].append(READ_FAILED.format(cause=exc))
|
| 337 |
return out
|
| 338 |
-
|
|
|
|
| 339 |
out["problems"].append(NOT_SYNCED)
|
| 340 |
return out
|
| 341 |
if not iso_period(facts["period"]):
|
|
@@ -391,7 +432,7 @@ def _rising_rows(cur, period, departments, prior, min_pct):
|
|
| 391 |
mark = _MARK.get()
|
| 392 |
key = (None if mark is None else
|
| 393 |
("rising", mark, period, tuple(departments), float(min_pct), hash(tuple(pairs))))
|
| 394 |
-
return _remember(key, lambda:
|
| 395 |
|
| 396 |
|
| 397 |
def _top3_list(raw, ours):
|
|
@@ -563,16 +604,19 @@ def _prior_week(st, period):
|
|
| 563 |
return before[-1], _doc(st, WEEK_PREFIX + before[-1]).get("terms") or {}
|
| 564 |
|
| 565 |
|
| 566 |
-
def find(tenant_key, *, st=None):
|
| 567 |
"""Every rising in-department term against the prior held week, sorted by rise. No cap (rule 1)."""
|
| 568 |
st = _rt(tenant_key, st)
|
| 569 |
min_pct = settings(tenant_key, st=st)["min_rise_pct"]
|
| 570 |
-
with _reader(tenant_key, st) as cur:
|
| 571 |
src = _source(cur, st)
|
| 572 |
prior_week, prior = _prior_week(st, src["period"])
|
| 573 |
rows = []
|
| 574 |
if src["period"] and src["departments"] and prior:
|
| 575 |
-
|
|
|
|
|
|
|
|
|
|
| 576 |
problems = list(src["problems"])
|
| 577 |
if src["period"] and src["departments"] and not prior_week:
|
| 578 |
problems.append(NOT_READY if _index(st) else NO_HISTORY)
|
|
@@ -592,7 +636,7 @@ def find(tenant_key, *, st=None):
|
|
| 592 |
"reason": reason_for(int(before), int(now_rank), float(pct), bool(mine))})
|
| 593 |
return {"week": src["period"][:10] if src["period"] else None, "period": src["period"],
|
| 594 |
"priorWeek": prior_week, "departments": src["departments"], "derivedDepartments": src["derived"],
|
| 595 |
-
"windows": windows, "rising": found, "problems": problems}
|
| 596 |
|
| 597 |
|
| 598 |
def _windows(st, period, prior_week):
|
|
@@ -644,12 +688,12 @@ def _page(rows, cursor, limit):
|
|
| 644 |
return rows[start:end], (str(end) if end < len(rows) else None)
|
| 645 |
|
| 646 |
|
| 647 |
-
def items(tenant_key, user=None, *, status=None, cursor=None, limit=DEFAULT_LIMIT, st=None):
|
| 648 |
"""C4: rising terms and held opportunities, one page. Counts cover every item."""
|
| 649 |
if status is not None and status not in STATUSES:
|
| 650 |
raise ValueError("Status must be one of: " + ", ".join(STATUSES) + ".")
|
| 651 |
st = _rt(tenant_key, st)
|
| 652 |
-
found = find(tenant_key, st=st)
|
| 653 |
every = _all_items(st, found)
|
| 654 |
counts = {s: 0 for s in STATUSES}
|
| 655 |
for item in every:
|
|
@@ -658,7 +702,8 @@ def items(tenant_key, user=None, *, status=None, cursor=None, limit=DEFAULT_LIMI
|
|
| 658 |
page, nxt = _page(wanted, cursor, limit)
|
| 659 |
return {"items": page, "cursor": nxt, "counts": counts, "total": len(wanted), "week": found["week"],
|
| 660 |
"priorWeek": found["priorWeek"], "departments": found["departments"],
|
| 661 |
-
"problems": found["problems"], "weeks": _index(st), "windows": found["windows"]
|
|
|
|
| 662 |
|
| 663 |
|
| 664 |
def _lookup(st, iid, found=None):
|
|
@@ -678,13 +723,13 @@ def _lookup(st, iid, found=None):
|
|
| 678 |
return fields, state if isinstance(state, dict) else None, term
|
| 679 |
|
| 680 |
|
| 681 |
-
def set_status(tenant_key, user, iid, status, *, note=None, owner=None, due=None, st=None):
|
| 682 |
"""C4: move one item, optionally its owner and due date. Every change appends a history event."""
|
| 683 |
st = _rt(tenant_key, st)
|
| 684 |
-
return _move(tenant_key, st, _editor(st, user), iid, status, note=note, owner=owner, due=due)
|
| 685 |
|
| 686 |
|
| 687 |
-
def _move(tenant_key, st, who, iid, status, *, note=None, owner=None, due=None):
|
| 688 |
if status not in STATUSES:
|
| 689 |
raise ValueError("Status must be one of: " + ", ".join(STATUSES) + ".")
|
| 690 |
if due is not None:
|
|
@@ -692,7 +737,7 @@ def _move(tenant_key, st, who, iid, status, *, note=None, owner=None, due=None):
|
|
| 692 |
due = dt.date.fromisoformat(_text(due)[:10]).isoformat()
|
| 693 |
except ValueError:
|
| 694 |
raise ValueError("Due must be a date such as 2026-10-30.") from None
|
| 695 |
-
fields, state, term = _lookup(st, iid, find(tenant_key, st=st))
|
| 696 |
if fields is None and state is None:
|
| 697 |
raise UnknownItem(UNKNOWN_ITEM)
|
| 698 |
owner = _text(owner) or None
|
|
@@ -753,25 +798,25 @@ def on_decision(tenant_key, iid, decision, reason, *, st=None):
|
|
| 753 |
return _move(tenant_key, st, "Amazon operator", iid, DECISIONS[decision], note=_text(reason) or None)
|
| 754 |
|
| 755 |
|
| 756 |
-
def item_detail(tenant_key, user, iid, *, st=None):
|
| 757 |
"""One item with its history, its weekly rank trend and its newest top 3."""
|
| 758 |
st = _rt(tenant_key, st)
|
| 759 |
-
found = find(tenant_key, st=st)
|
| 760 |
fields, state, term = _lookup(st, iid, found)
|
| 761 |
row = next((r for r in _all_items(st, found) if r["id"] == iid), None)
|
| 762 |
if row is None:
|
| 763 |
row = _row({"id": iid, "term": term, "title": term, "rising": False, "reason": NOT_RISING,
|
| 764 |
"asin": None, "top3": []}, state or {})
|
| 765 |
return {"item": row, "history": list((state or {}).get("history") or ()),
|
| 766 |
-
**term_detail(tenant_key, iid, st=st, _term=term, _found=found)}
|
| 767 |
|
| 768 |
|
| 769 |
-
def term_detail(tenant_key, iid, *, st=None, _term=None, _found=None):
|
| 770 |
"""The weekly rank trend from history, oldest first, plus the newest week's top 3 clicked ASINs."""
|
| 771 |
st = _rt(tenant_key, st)
|
| 772 |
term = _term
|
| 773 |
if term is None:
|
| 774 |
-
_fields, _state, term = _lookup(st, iid, _found or find(tenant_key, st=st))
|
| 775 |
if term is None:
|
| 776 |
raise UnknownItem(UNKNOWN_ITEM)
|
| 777 |
trend = []
|
|
@@ -780,7 +825,7 @@ def term_detail(tenant_key, iid, *, st=None, _term=None, _found=None):
|
|
| 780 |
if term in terms:
|
| 781 |
trend.append({"week": week, "rank": terms[term]})
|
| 782 |
top3, period = [], None
|
| 783 |
-
with _reader(tenant_key, st) as cur:
|
| 784 |
src = _source(cur, st)
|
| 785 |
period = src["period"]
|
| 786 |
if period and src["departments"] and {"clicked_asin", "click_share_rank"} <= _columns(cur, BA_TABLE):
|
|
@@ -789,19 +834,19 @@ def term_detail(tenant_key, iid, *, st=None, _term=None, _found=None):
|
|
| 789 |
share = 'TRY_CAST(trim(CAST("click_share" AS VARCHAR)) AS DOUBLE)' if "click_share" in cols else "NULL"
|
| 790 |
conv = ('TRY_CAST(trim(CAST("conversion_share" AS VARCHAR)) AS DOUBLE)'
|
| 791 |
if "conversion_share" in cols else "NULL")
|
| 792 |
-
rows =
|
| 793 |
-
f'SELECT trim(CAST("clicked_asin" AS VARCHAR)), {pos}, {share}, {conv} FROM {BA_TABLE} '
|
| 794 |
f'WHERE {_IN_PERIOD} AND {_IN_DEPTS} AND {_TERM} = ? AND {pos} BETWEEN 1 AND 3 '
|
| 795 |
-
f'ORDER BY 2', [period, src["departments"], term])
|
| 796 |
top3 = _top3_list([{"asin": a, "pos": p, "share": s, "conv": c} for a, p, s, c in rows], src["ours"])
|
| 797 |
return {"id": iid, "term": term, "trend": trend, "top3": top3,
|
| 798 |
"week": period[:10] if period else None}
|
| 799 |
|
| 800 |
|
| 801 |
-
def validate(tenant_key, *, week=None, start=None, end=None, st=None):
|
| 802 |
"""Rule 8: the stored week's term count against a SQL distinct count of the mirror week, our departments."""
|
| 803 |
st = _rt(tenant_key, st)
|
| 804 |
-
with _reader(tenant_key, st) as cur:
|
| 805 |
src = _source(cur, st, fresh=True)
|
| 806 |
period = src["period"]
|
| 807 |
if period is None:
|
|
@@ -816,9 +861,9 @@ def validate(tenant_key, *, week=None, start=None, end=None, st=None):
|
|
| 816 |
"detail": OLDER_WEEK}
|
| 817 |
independent = 0
|
| 818 |
if src["departments"]:
|
| 819 |
-
independent = int(
|
| 820 |
-
f'SELECT COUNT(DISTINCT {_TERM}) FROM {BA_TABLE} WHERE {_IN_PERIOD} AND {_IN_DEPTS}',
|
| 821 |
-
[period, src["departments"]])
|
| 822 |
doc = _doc(st, WEEK_PREFIX + newest)
|
| 823 |
if not doc:
|
| 824 |
return {"ok": False, "app_total": None, "independent_total": independent, "week": newest,
|
|
|
|
| 22 |
import contextvars
|
| 23 |
import datetime as dt
|
| 24 |
import hashlib
|
| 25 |
+
import logging
|
| 26 |
import os
|
| 27 |
import threading
|
| 28 |
from pathlib import Path
|
|
|
|
| 69 |
|
| 70 |
#: A mirror probe longer than this is interrupted and reads as busy, under the 45 s request deadline.
|
| 71 |
PROBE_SECONDS = 20.0
|
| 72 |
+
#: A fired probe bound repeats interrupt() this often: DuckDB can lose a single one on a parameterized execute.
|
| 73 |
+
INTERRUPT_EVERY = 0.05
|
| 74 |
#: Mirror facts and rising lists kept per process, keyed on the mirror mark. Oldest goes first.
|
| 75 |
MEMO_KEEP = 16
|
| 76 |
|
| 77 |
#: Accounts that sign in for a process, never a person who can own an item.
|
| 78 |
SERVICE_ACCOUNTS = ("admin", "automation", "scheduler", "qa-runner", "leadership")
|
| 79 |
|
| 80 |
+
LOG = logging.getLogger("aios.opportunity_finder")
|
| 81 |
+
|
| 82 |
|
| 83 |
class UnknownItem(KeyError, ValueError):
|
| 84 |
"""An item id the finder does not hold. A KeyError for the router, a ValueError for C4."""
|
|
|
|
| 150 |
_MEMO_LOCK = threading.Lock()
|
| 151 |
|
| 152 |
|
| 153 |
+
def _remember(key, build, keep=None):
|
| 154 |
+
"""`build()` once per key. A failed build, or one `keep` refuses, is not kept: the next request reads again."""
|
| 155 |
if key is None:
|
| 156 |
return build()
|
| 157 |
with _MEMO_LOCK:
|
| 158 |
if key in _MEMO:
|
| 159 |
return _MEMO[key]
|
| 160 |
value = build()
|
| 161 |
+
if keep is not None and not keep(value):
|
| 162 |
+
return value
|
| 163 |
with _MEMO_LOCK:
|
| 164 |
_MEMO[key] = value
|
| 165 |
while len(_MEMO) > MEMO_KEEP:
|
|
|
|
| 167 |
return value
|
| 168 |
|
| 169 |
|
| 170 |
+
def _mark(cur, shared):
|
| 171 |
+
"""Moves whenever the mirror rows can have changed: an install, a version swap or a rewrite.
|
| 172 |
+
|
| 173 |
+
The file is the one the cursor reads, so a reader that followed the serving pointer to
|
| 174 |
+
`<stem>.v<N>.duckdb` gets a new mark, even where the logical path and its stats stay put."""
|
| 175 |
from harness import datastore
|
| 176 |
+
rows = _bounded(cur, "SELECT path FROM duckdb_databases() WHERE database_name = current_database()")
|
| 177 |
+
path = rows[0][0] if rows and rows[0][0] else ""
|
| 178 |
generation = None
|
| 179 |
if shared:
|
| 180 |
with contextlib.suppress(Exception):
|
| 181 |
+
generation = (datastore.mirror_generation(), str(datastore.serving_version().get("version")))
|
| 182 |
stamps = []
|
| 183 |
for part in (Path(path), Path(str(path) + ".wal")):
|
| 184 |
try:
|
|
|
|
| 190 |
|
| 191 |
|
| 192 |
@contextlib.contextmanager
|
| 193 |
+
def _reader(tenant_key, st, opener=None):
|
| 194 |
"""A read cursor on the tenant mirror, None when there is no mirror, `_Unreadable` when it cannot open.
|
| 195 |
|
| 196 |
+
`opener` is the typed `ut_amzn_*` door the route hands in (modules never import api): it asserts the
|
| 197 |
+
tenant and returns `datastore.ro_cursor()` on the serving mirror version. Without it, the shared
|
| 198 |
+
instance serves when it is this mirror, else the file the mirror's serving pointer names."""
|
| 199 |
from harness import datastore
|
| 200 |
path = Path(_mirror_path(tenant_key, st))
|
| 201 |
try:
|
| 202 |
shared = Path(datastore.DB_PATH).resolve() == path.resolve()
|
| 203 |
except Exception: # noqa: BLE001
|
| 204 |
shared = False
|
| 205 |
+
try:
|
| 206 |
+
physical = path if shared or opener is not None else Path(datastore.serving_file(path)[0])
|
| 207 |
+
present = physical.exists() or (opener is not None and Path(datastore.serving_file()[0]).exists())
|
| 208 |
+
except Exception: # noqa: BLE001
|
| 209 |
+
physical, present = path, path.exists()
|
| 210 |
+
if not present:
|
| 211 |
yield None
|
| 212 |
return
|
| 213 |
cur = None
|
| 214 |
try:
|
| 215 |
+
if opener is not None:
|
| 216 |
+
cur = opener()
|
| 217 |
+
elif shared:
|
| 218 |
if hasattr(st, "assert_datastore_matches"):
|
| 219 |
st.assert_datastore_matches()
|
| 220 |
cur = datastore.ro_cursor()
|
| 221 |
else:
|
| 222 |
import duckdb
|
| 223 |
+
cur = duckdb.connect(str(physical), read_only=True)
|
| 224 |
+
mark = _mark(cur, shared or opener is not None)
|
| 225 |
except Exception as exc: # noqa: BLE001
|
| 226 |
if cur is not None:
|
| 227 |
with contextlib.suppress(Exception):
|
| 228 |
cur.close()
|
| 229 |
+
yield _Unreadable(str(exc) if isinstance(exc, _ReadFailed) else type(exc).__name__)
|
| 230 |
return
|
| 231 |
+
token = _MARK.set(mark)
|
| 232 |
try:
|
| 233 |
yield cur
|
| 234 |
finally:
|
|
|
|
| 242 |
|
| 243 |
|
| 244 |
def _bounded(cur, sql, params=()):
|
| 245 |
+
"""Every row of one query, interrupted past PROBE_SECONDS. Any failure raises `_ReadFailed`.
|
| 246 |
|
| 247 |
+
Past the bound, interrupt() repeats every INTERRUPT_EVERY until the query ends: one call can be lost."""
|
| 248 |
if cur is None or isinstance(cur, _Unreadable):
|
| 249 |
raise _ReadFailed(getattr(cur, "cause", "NoMirror"))
|
| 250 |
+
lock, done = threading.Lock(), threading.Event()
|
| 251 |
+
|
| 252 |
+
def _fire():
|
| 253 |
+
while True:
|
| 254 |
+
with lock:
|
| 255 |
+
if done.is_set():
|
| 256 |
+
return
|
| 257 |
+
_interrupt(cur)
|
| 258 |
+
if done.wait(INTERRUPT_EVERY):
|
| 259 |
+
return
|
| 260 |
+
timer = threading.Timer(PROBE_SECONDS, _fire)
|
| 261 |
timer.daemon = True
|
| 262 |
timer.start()
|
| 263 |
try:
|
|
|
|
| 265 |
except Exception as exc: # noqa: BLE001
|
| 266 |
raise _ReadFailed(type(exc).__name__) from exc
|
| 267 |
finally:
|
| 268 |
+
with lock:
|
| 269 |
+
done.set()
|
| 270 |
timer.cancel()
|
| 271 |
|
| 272 |
|
|
|
|
| 274 |
"""{column: data type} of a mirror table, {} when the table is absent. A failed read raises."""
|
| 275 |
if not table.replace("_", "").isalnum():
|
| 276 |
raise ValueError(f"{table!r} is not a table name.")
|
|
|
|
| 277 |
return {str(n): str(t) for n, t in _bounded(
|
| 278 |
cur, f"SELECT column_name, data_type FROM information_schema.columns WHERE table_name = '{table}'")}
|
| 279 |
|
| 280 |
|
| 281 |
def _columns(cur, table):
|
| 282 |
+
"""The column names of a mirror table, empty when the table is absent. A failed read raises `_ReadFailed`."""
|
| 283 |
+
return set(_types(cur, table))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 284 |
|
| 285 |
|
| 286 |
def iso_period(value):
|
|
|
|
| 296 |
"""SQP ASINs plus listing `asin1`: the ASINs the seller owns."""
|
| 297 |
ours = set()
|
| 298 |
if "asin" in _columns(cur, SQP_TABLE):
|
| 299 |
+
ours |= {_text(r[0]) for r in _bounded(
|
| 300 |
+
cur, f'SELECT DISTINCT CAST("asin" AS VARCHAR) FROM {SQP_TABLE} WHERE "asin" IS NOT NULL')}
|
| 301 |
if "asin1" in _columns(cur, LISTINGS_TABLE):
|
| 302 |
+
ours |= {_text(r[0]) for r in _bounded(
|
| 303 |
+
cur, f'SELECT DISTINCT CAST("asin1" AS VARCHAR) FROM {LISTINGS_TABLE} WHERE "asin1" IS NOT NULL')}
|
| 304 |
return {a for a in ours if a}
|
| 305 |
|
| 306 |
|
|
|
|
| 324 |
"""R49 default: every department where a brand analytics row clicked one of our ASINs, any period."""
|
| 325 |
if not ours or "clicked_asin" not in _columns(cur, BA_TABLE):
|
| 326 |
return []
|
| 327 |
+
rows = _bounded(
|
| 328 |
+
cur, f'SELECT DISTINCT {_DEPT} FROM {BA_TABLE} '
|
| 329 |
f'WHERE trim(CAST("clicked_asin" AS VARCHAR)) IN (SELECT unnest(?::VARCHAR[])) '
|
| 330 |
+
f'AND "department_name" IS NOT NULL', [sorted(ours)])
|
| 331 |
return sorted(_text(r[0]) for r in rows if _text(r[0]))
|
| 332 |
|
| 333 |
|
| 334 |
+
def _synced(facts):
|
| 335 |
+
return set(BA_NEED) <= facts["columns"] and facts["period"] is not None
|
| 336 |
+
|
| 337 |
+
|
| 338 |
def _facts(cur, fresh=False):
|
| 339 |
+
"""The mirror facts every read starts from, once per mirror mark: columns, newest period, ours, derived.
|
| 340 |
+
|
| 341 |
+
A not-synced answer is never kept, so a mirror that gains the table is read at the next request."""
|
| 342 |
def build():
|
| 343 |
types = _types(cur, BA_TABLE)
|
| 344 |
+
facts = {"columns": set(types), "period": None, "ours": set(), "derived": [],
|
| 345 |
+
"period_type": types.get("_period_start")}
|
| 346 |
if not set(BA_NEED) <= facts["columns"]:
|
| 347 |
return facts
|
| 348 |
facts["period"] = _period(cur, types.get("_period_start", "VARCHAR"))
|
|
|
|
| 351 |
try:
|
| 352 |
facts["ours"] = _our_asins(cur)
|
| 353 |
facts["derived"] = _derived_departments(cur, facts["ours"])
|
| 354 |
+
except _ReadFailed:
|
| 355 |
+
raise
|
| 356 |
except Exception as exc: # noqa: BLE001
|
| 357 |
raise _ReadFailed(type(exc).__name__) from exc
|
| 358 |
return facts
|
| 359 |
mark = _MARK.get()
|
| 360 |
+
facts = _remember(None if mark is None or fresh else ("facts", mark), build, keep=_synced)
|
| 361 |
+
if not _synced(facts):
|
| 362 |
+
LOG.warning("[opportunity-finder] %s not synced: file %s, missing columns %s, newest period %r",
|
| 363 |
+
BA_TABLE, mark[0] if mark else "?", sorted(set(BA_NEED) - facts["columns"]), facts["period"])
|
| 364 |
+
return facts
|
| 365 |
|
| 366 |
|
| 367 |
def _source(cur, st, fresh=False):
|
| 368 |
"""What the mirror holds now: {period, departments, ours, problems}. Departments follow the setting."""
|
| 369 |
+
out = {"period": None, "departments": [], "ours": set(), "problems": [], "derived": [], "periodType": None}
|
| 370 |
if cur is None:
|
| 371 |
out["problems"].append(NO_MIRROR)
|
| 372 |
return out
|
|
|
|
| 375 |
except _ReadFailed as exc:
|
| 376 |
out["problems"].append(READ_FAILED.format(cause=exc))
|
| 377 |
return out
|
| 378 |
+
out["periodType"] = facts.get("period_type")
|
| 379 |
+
if not _synced(facts):
|
| 380 |
out["problems"].append(NOT_SYNCED)
|
| 381 |
return out
|
| 382 |
if not iso_period(facts["period"]):
|
|
|
|
| 432 |
mark = _MARK.get()
|
| 433 |
key = (None if mark is None else
|
| 434 |
("rising", mark, period, tuple(departments), float(min_pct), hash(tuple(pairs))))
|
| 435 |
+
return _remember(key, lambda: _bounded(cur, sql, params))
|
| 436 |
|
| 437 |
|
| 438 |
def _top3_list(raw, ours):
|
|
|
|
| 604 |
return before[-1], _doc(st, WEEK_PREFIX + before[-1]).get("terms") or {}
|
| 605 |
|
| 606 |
|
| 607 |
+
def find(tenant_key, *, st=None, opener=None):
|
| 608 |
"""Every rising in-department term against the prior held week, sorted by rise. No cap (rule 1)."""
|
| 609 |
st = _rt(tenant_key, st)
|
| 610 |
min_pct = settings(tenant_key, st=st)["min_rise_pct"]
|
| 611 |
+
with _reader(tenant_key, st, opener) as cur:
|
| 612 |
src = _source(cur, st)
|
| 613 |
prior_week, prior = _prior_week(st, src["period"])
|
| 614 |
rows = []
|
| 615 |
if src["period"] and src["departments"] and prior:
|
| 616 |
+
try:
|
| 617 |
+
rows = _rising_rows(cur, src["period"], src["departments"], prior, min_pct)
|
| 618 |
+
except _ReadFailed as exc:
|
| 619 |
+
src["problems"].append(READ_FAILED.format(cause=exc))
|
| 620 |
problems = list(src["problems"])
|
| 621 |
if src["period"] and src["departments"] and not prior_week:
|
| 622 |
problems.append(NOT_READY if _index(st) else NO_HISTORY)
|
|
|
|
| 636 |
"reason": reason_for(int(before), int(now_rank), float(pct), bool(mine))})
|
| 637 |
return {"week": src["period"][:10] if src["period"] else None, "period": src["period"],
|
| 638 |
"priorWeek": prior_week, "departments": src["departments"], "derivedDepartments": src["derived"],
|
| 639 |
+
"windows": windows, "periodType": src["periodType"], "rising": found, "problems": problems}
|
| 640 |
|
| 641 |
|
| 642 |
def _windows(st, period, prior_week):
|
|
|
|
| 688 |
return rows[start:end], (str(end) if end < len(rows) else None)
|
| 689 |
|
| 690 |
|
| 691 |
+
def items(tenant_key, user=None, *, status=None, cursor=None, limit=DEFAULT_LIMIT, st=None, opener=None):
|
| 692 |
"""C4: rising terms and held opportunities, one page. Counts cover every item."""
|
| 693 |
if status is not None and status not in STATUSES:
|
| 694 |
raise ValueError("Status must be one of: " + ", ".join(STATUSES) + ".")
|
| 695 |
st = _rt(tenant_key, st)
|
| 696 |
+
found = find(tenant_key, st=st, opener=opener)
|
| 697 |
every = _all_items(st, found)
|
| 698 |
counts = {s: 0 for s in STATUSES}
|
| 699 |
for item in every:
|
|
|
|
| 702 |
page, nxt = _page(wanted, cursor, limit)
|
| 703 |
return {"items": page, "cursor": nxt, "counts": counts, "total": len(wanted), "week": found["week"],
|
| 704 |
"priorWeek": found["priorWeek"], "departments": found["departments"],
|
| 705 |
+
"problems": found["problems"], "weeks": _index(st), "windows": found["windows"],
|
| 706 |
+
"periodType": found["periodType"]}
|
| 707 |
|
| 708 |
|
| 709 |
def _lookup(st, iid, found=None):
|
|
|
|
| 723 |
return fields, state if isinstance(state, dict) else None, term
|
| 724 |
|
| 725 |
|
| 726 |
+
def set_status(tenant_key, user, iid, status, *, note=None, owner=None, due=None, st=None, opener=None):
|
| 727 |
"""C4: move one item, optionally its owner and due date. Every change appends a history event."""
|
| 728 |
st = _rt(tenant_key, st)
|
| 729 |
+
return _move(tenant_key, st, _editor(st, user), iid, status, note=note, owner=owner, due=due, opener=opener)
|
| 730 |
|
| 731 |
|
| 732 |
+
def _move(tenant_key, st, who, iid, status, *, note=None, owner=None, due=None, opener=None):
|
| 733 |
if status not in STATUSES:
|
| 734 |
raise ValueError("Status must be one of: " + ", ".join(STATUSES) + ".")
|
| 735 |
if due is not None:
|
|
|
|
| 737 |
due = dt.date.fromisoformat(_text(due)[:10]).isoformat()
|
| 738 |
except ValueError:
|
| 739 |
raise ValueError("Due must be a date such as 2026-10-30.") from None
|
| 740 |
+
fields, state, term = _lookup(st, iid, find(tenant_key, st=st, opener=opener))
|
| 741 |
if fields is None and state is None:
|
| 742 |
raise UnknownItem(UNKNOWN_ITEM)
|
| 743 |
owner = _text(owner) or None
|
|
|
|
| 798 |
return _move(tenant_key, st, "Amazon operator", iid, DECISIONS[decision], note=_text(reason) or None)
|
| 799 |
|
| 800 |
|
| 801 |
+
def item_detail(tenant_key, user, iid, *, st=None, opener=None):
|
| 802 |
"""One item with its history, its weekly rank trend and its newest top 3."""
|
| 803 |
st = _rt(tenant_key, st)
|
| 804 |
+
found = find(tenant_key, st=st, opener=opener)
|
| 805 |
fields, state, term = _lookup(st, iid, found)
|
| 806 |
row = next((r for r in _all_items(st, found) if r["id"] == iid), None)
|
| 807 |
if row is None:
|
| 808 |
row = _row({"id": iid, "term": term, "title": term, "rising": False, "reason": NOT_RISING,
|
| 809 |
"asin": None, "top3": []}, state or {})
|
| 810 |
return {"item": row, "history": list((state or {}).get("history") or ()),
|
| 811 |
+
**term_detail(tenant_key, iid, st=st, _term=term, _found=found, opener=opener)}
|
| 812 |
|
| 813 |
|
| 814 |
+
def term_detail(tenant_key, iid, *, st=None, _term=None, _found=None, opener=None):
|
| 815 |
"""The weekly rank trend from history, oldest first, plus the newest week's top 3 clicked ASINs."""
|
| 816 |
st = _rt(tenant_key, st)
|
| 817 |
term = _term
|
| 818 |
if term is None:
|
| 819 |
+
_fields, _state, term = _lookup(st, iid, _found or find(tenant_key, st=st, opener=opener))
|
| 820 |
if term is None:
|
| 821 |
raise UnknownItem(UNKNOWN_ITEM)
|
| 822 |
trend = []
|
|
|
|
| 825 |
if term in terms:
|
| 826 |
trend.append({"week": week, "rank": terms[term]})
|
| 827 |
top3, period = [], None
|
| 828 |
+
with _reader(tenant_key, st, opener) as cur:
|
| 829 |
src = _source(cur, st)
|
| 830 |
period = src["period"]
|
| 831 |
if period and src["departments"] and {"clicked_asin", "click_share_rank"} <= _columns(cur, BA_TABLE):
|
|
|
|
| 834 |
share = 'TRY_CAST(trim(CAST("click_share" AS VARCHAR)) AS DOUBLE)' if "click_share" in cols else "NULL"
|
| 835 |
conv = ('TRY_CAST(trim(CAST("conversion_share" AS VARCHAR)) AS DOUBLE)'
|
| 836 |
if "conversion_share" in cols else "NULL")
|
| 837 |
+
rows = _bounded(
|
| 838 |
+
cur, f'SELECT trim(CAST("clicked_asin" AS VARCHAR)), {pos}, {share}, {conv} FROM {BA_TABLE} '
|
| 839 |
f'WHERE {_IN_PERIOD} AND {_IN_DEPTS} AND {_TERM} = ? AND {pos} BETWEEN 1 AND 3 '
|
| 840 |
+
f'ORDER BY 2', [period, src["departments"], term])
|
| 841 |
top3 = _top3_list([{"asin": a, "pos": p, "share": s, "conv": c} for a, p, s, c in rows], src["ours"])
|
| 842 |
return {"id": iid, "term": term, "trend": trend, "top3": top3,
|
| 843 |
"week": period[:10] if period else None}
|
| 844 |
|
| 845 |
|
| 846 |
+
def validate(tenant_key, *, week=None, start=None, end=None, st=None, opener=None):
|
| 847 |
"""Rule 8: the stored week's term count against a SQL distinct count of the mirror week, our departments."""
|
| 848 |
st = _rt(tenant_key, st)
|
| 849 |
+
with _reader(tenant_key, st, opener) as cur:
|
| 850 |
src = _source(cur, st, fresh=True)
|
| 851 |
period = src["period"]
|
| 852 |
if period is None:
|
|
|
|
| 861 |
"detail": OLDER_WEEK}
|
| 862 |
independent = 0
|
| 863 |
if src["departments"]:
|
| 864 |
+
independent = int(_bounded(
|
| 865 |
+
cur, f'SELECT COUNT(DISTINCT {_TERM}) FROM {BA_TABLE} WHERE {_IN_PERIOD} AND {_IN_DEPTS}',
|
| 866 |
+
[period, src["departments"]])[0][0])
|
| 867 |
doc = _doc(st, WEEK_PREFIX + newest)
|
| 868 |
if not doc:
|
| 869 |
return {"ok": False, "app_total": None, "independent_total": independent, "week": newest,
|
platform/modules/ranking_diagnostic.py
CHANGED
|
@@ -85,6 +85,9 @@ CAUSE_ORDER = {key: n for n, (key, _l, _f) in enumerate((*CAUSES, UNEXPLAINED))}
|
|
| 85 |
ABSENT = "Not in this week's Sales and Traffic report."
|
| 86 |
READ_ONLY = "Your account can view the ranking diagnostic but not change it."
|
| 87 |
NOT_READY = "The diagnostic starts after two weekly syncs. One week is held so far."
|
|
|
|
|
|
|
|
|
|
| 88 |
LIMIT = ("Each week of history is one store document, limited to 128 MB. About 300,000 ASINs fit in "
|
| 89 |
"one week. Split a week by ASIN range when a catalogue nears that size.")
|
| 90 |
|
|
@@ -155,9 +158,15 @@ def _mirror_path(tenant_key, st):
|
|
| 155 |
return amazon_store._tenant_mirror_path(tenant_key, rt)
|
| 156 |
|
| 157 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 158 |
@contextlib.contextmanager
|
| 159 |
def _reader(tenant_key, st):
|
| 160 |
-
"""A read-only cursor on the tenant mirror,
|
|
|
|
|
|
|
| 161 |
path = Path(_mirror_path(tenant_key, st))
|
| 162 |
if not path.exists():
|
| 163 |
yield None
|
|
@@ -168,29 +177,29 @@ def _reader(tenant_key, st):
|
|
| 168 |
except Exception: # noqa: BLE001
|
| 169 |
shared = False
|
| 170 |
# The process holds one writer on its own mirror: a separate read-only connect would refuse.
|
| 171 |
-
if shared:
|
| 172 |
-
cur = datastore.ro_cursor()
|
| 173 |
-
try:
|
| 174 |
-
yield cur
|
| 175 |
-
finally:
|
| 176 |
-
cur.close()
|
| 177 |
-
return
|
| 178 |
-
import duckdb
|
| 179 |
-
con = duckdb.connect(str(path), read_only=True)
|
| 180 |
try:
|
| 181 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 182 |
finally:
|
| 183 |
-
|
| 184 |
|
| 185 |
|
| 186 |
def _columns(cur, table):
|
|
|
|
| 187 |
if cur is None:
|
| 188 |
return set()
|
| 189 |
try:
|
| 190 |
return {r[0] for r in cur.execute(
|
| 191 |
"SELECT column_name FROM information_schema.columns WHERE table_name = ?", [table]).fetchall()}
|
| 192 |
-
except Exception:
|
| 193 |
-
|
| 194 |
|
| 195 |
|
| 196 |
def _newest(cols, column, table):
|
|
@@ -254,6 +263,7 @@ def _pricing(cur):
|
|
| 254 |
for a, doc, _at, sku, sessions, units, sales, box in rows:
|
| 255 |
if not a or newest[a] != (landed[doc], doc):
|
| 256 |
continue
|
|
|
|
| 257 |
p = out.setdefault(a, {"sessions": 0.0, "units": 0.0, "sales": 0.0, "skus": [], "_box": []})
|
| 258 |
p["sessions"] += sessions or 0.0
|
| 259 |
p["units"] += units or 0.0
|
|
@@ -275,12 +285,14 @@ def source_totals(cur):
|
|
| 275 |
"""P10: sessions and units in SQL over each ASIN's newest document, apart from `_pricing`'s Python pick.
|
| 276 |
|
| 277 |
Both apply one rule (newest document per ASIN), so the tie proves the two picks and sums agree.
|
| 278 |
-
`supersededRows` and the raw sums show what the rule left out.
|
|
|
|
| 279 |
cols = _columns(cur, PRICING)
|
| 280 |
asin = _asin_expr(cols)
|
| 281 |
if asin is None or not {SESSIONS, UNITS} <= cols:
|
| 282 |
return None
|
| 283 |
-
|
|
|
|
| 284 |
f"WITH r AS (SELECT {asin} AS a, {_document(cols)} AS d, {_landing(cur)} AS i, "
|
| 285 |
f"TRY_CAST(\"{SESSIONS}\" AS DOUBLE) AS s, TRY_CAST(\"{UNITS}\" AS DOUBLE) AS u FROM {PRICING}), "
|
| 286 |
f"docs AS (SELECT d, MAX(i) AS landed FROM r GROUP BY d), "
|
|
@@ -289,10 +301,11 @@ def source_totals(cur):
|
|
| 289 |
f"WHERE n = 1), kept AS (SELECT r.* FROM r JOIN best USING (a, d)) "
|
| 290 |
f"SELECT COALESCE(SUM(s), 0), COALESCE(SUM(u), 0), COUNT(DISTINCT CASE WHEN u > 0 THEN a END), "
|
| 291 |
f"(SELECT COUNT(*) FROM r WHERE a IS NOT NULL) - COUNT(*), (SELECT COALESCE(SUM(s), 0) FROM r), "
|
| 292 |
-
f"(SELECT COALESCE(SUM(u), 0) FROM r)
|
|
|
|
| 293 |
return {"sessions": round(float(sessions), 4), "units": round(float(units), 4), "asinsWithUnits": int(count),
|
| 294 |
"supersededRows": int(superseded), "rawSessions": round(float(raw_sessions), 4),
|
| 295 |
-
"rawUnits": round(float(raw_units), 4)}
|
| 296 |
|
| 297 |
|
| 298 |
def _rank(raw):
|
|
@@ -724,7 +737,8 @@ def on_amazon_sync(tenant_key, synced_tables, *, st=None, now=None):
|
|
| 724 |
try:
|
| 725 |
def _note(doc):
|
| 726 |
doc = doc if isinstance(doc, dict) else {}
|
| 727 |
-
doc["last_error"] =
|
|
|
|
| 728 |
return doc
|
| 729 |
_rt(tenant_key, st).update(KEY_INDEX, _note)
|
| 730 |
except Exception: # noqa: BLE001
|
|
@@ -890,10 +904,14 @@ def validate(tenant_key, *, week=None, start=None, end=None, st=None):
|
|
| 890 |
if not chosen:
|
| 891 |
return {"ok": False, "app_total": None, "independent_total": None, "weeks": [],
|
| 892 |
"detail": "No weekly history is held for that period, so nothing can be tied."}
|
| 893 |
-
live = None
|
| 894 |
if held and held[-1] in chosen:
|
| 895 |
-
|
| 896 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 897 |
rows, ok = [], True
|
| 898 |
app = {"sessions": 0.0, "units": 0.0}
|
| 899 |
ind = {"sessions": 0.0, "units": 0.0}
|
|
@@ -915,6 +933,8 @@ def validate(tenant_key, *, week=None, start=None, end=None, st=None):
|
|
| 915 |
bad = [r["week"] for r in rows if not r["ok"]]
|
| 916 |
detail = (f"{len(rows)} weeks tie sessions and units to the Sales and Traffic totals." if ok else
|
| 917 |
"These weeks do not tie to the Sales and Traffic totals: " + ", ".join(bad) + ".")
|
|
|
|
|
|
|
| 918 |
superseded = int((live or {}).get("supersededRows") or 0)
|
| 919 |
if superseded:
|
| 920 |
detail += (f" {superseded} older Sales and Traffic rows are left out because a newer report covers "
|
|
|
|
| 85 |
ABSENT = "Not in this week's Sales and Traffic report."
|
| 86 |
READ_ONLY = "Your account can view the ranking diagnostic but not change it."
|
| 87 |
NOT_READY = "The diagnostic starts after two weekly syncs. One week is held so far."
|
| 88 |
+
#: The mirror holds the tables but this read failed or was busy. `{cause}` is the error class.
|
| 89 |
+
READ_FAILED = ("The Amazon mirror could not be read just now ({cause}), so the weekly history was not updated. "
|
| 90 |
+
"The next Amazon sync reads it again.")
|
| 91 |
LIMIT = ("Each week of history is one store document, limited to 128 MB. About 300,000 ASINs fit in "
|
| 92 |
"one week. Split a week by ASIN range when a catalogue nears that size.")
|
| 93 |
|
|
|
|
| 158 |
return amazon_store._tenant_mirror_path(tenant_key, rt)
|
| 159 |
|
| 160 |
|
| 161 |
+
class _ReadFailed(Exception):
|
| 162 |
+
"""A mirror read failed, was locked or busy. Never read as an absent table."""
|
| 163 |
+
|
| 164 |
+
|
| 165 |
@contextlib.contextmanager
|
| 166 |
def _reader(tenant_key, st):
|
| 167 |
+
"""A read-only cursor on the tenant mirror, None when the mirror file does not exist.
|
| 168 |
+
|
| 169 |
+
An open failure raises `_ReadFailed` naming its error class."""
|
| 170 |
path = Path(_mirror_path(tenant_key, st))
|
| 171 |
if not path.exists():
|
| 172 |
yield None
|
|
|
|
| 177 |
except Exception: # noqa: BLE001
|
| 178 |
shared = False
|
| 179 |
# The process holds one writer on its own mirror: a separate read-only connect would refuse.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 180 |
try:
|
| 181 |
+
if shared:
|
| 182 |
+
cur = datastore.ro_cursor()
|
| 183 |
+
else:
|
| 184 |
+
import duckdb
|
| 185 |
+
cur = duckdb.connect(str(path), read_only=True)
|
| 186 |
+
except Exception as exc: # noqa: BLE001
|
| 187 |
+
raise _ReadFailed(type(exc).__name__) from exc
|
| 188 |
+
try:
|
| 189 |
+
yield cur
|
| 190 |
finally:
|
| 191 |
+
cur.close()
|
| 192 |
|
| 193 |
|
| 194 |
def _columns(cur, table):
|
| 195 |
+
"""The table's column names, an empty set when it is absent. A failed read raises `_ReadFailed`."""
|
| 196 |
if cur is None:
|
| 197 |
return set()
|
| 198 |
try:
|
| 199 |
return {r[0] for r in cur.execute(
|
| 200 |
"SELECT column_name FROM information_schema.columns WHERE table_name = ?", [table]).fetchall()}
|
| 201 |
+
except Exception as exc: # noqa: BLE001
|
| 202 |
+
raise _ReadFailed(type(exc).__name__) from exc
|
| 203 |
|
| 204 |
|
| 205 |
def _newest(cols, column, table):
|
|
|
|
| 263 |
for a, doc, _at, sku, sessions, units, sales, box in rows:
|
| 264 |
if not a or newest[a] != (landed[doc], doc):
|
| 265 |
continue
|
| 266 |
+
# One child under two parents in one document is two traffic slices of that child: SUM, never pick.
|
| 267 |
p = out.setdefault(a, {"sessions": 0.0, "units": 0.0, "sales": 0.0, "skus": [], "_box": []})
|
| 268 |
p["sessions"] += sessions or 0.0
|
| 269 |
p["units"] += units or 0.0
|
|
|
|
| 285 |
"""P10: sessions and units in SQL over each ASIN's newest document, apart from `_pricing`'s Python pick.
|
| 286 |
|
| 287 |
Both apply one rule (newest document per ASIN), so the tie proves the two picks and sums agree.
|
| 288 |
+
`supersededRows` and the raw sums show what the rule left out. `summedAsins` counts the ASINs
|
| 289 |
+
whose kept document lists them on more than one row (one child under two parents)."""
|
| 290 |
cols = _columns(cur, PRICING)
|
| 291 |
asin = _asin_expr(cols)
|
| 292 |
if asin is None or not {SESSIONS, UNITS} <= cols:
|
| 293 |
return None
|
| 294 |
+
# Every kept row of an ASIN sums, the same rule as `_pricing`: a child under two parents counts once.
|
| 295 |
+
sessions, units, count, superseded, raw_sessions, raw_units, summed = cur.execute(
|
| 296 |
f"WITH r AS (SELECT {asin} AS a, {_document(cols)} AS d, {_landing(cur)} AS i, "
|
| 297 |
f"TRY_CAST(\"{SESSIONS}\" AS DOUBLE) AS s, TRY_CAST(\"{UNITS}\" AS DOUBLE) AS u FROM {PRICING}), "
|
| 298 |
f"docs AS (SELECT d, MAX(i) AS landed FROM r GROUP BY d), "
|
|
|
|
| 301 |
f"WHERE n = 1), kept AS (SELECT r.* FROM r JOIN best USING (a, d)) "
|
| 302 |
f"SELECT COALESCE(SUM(s), 0), COALESCE(SUM(u), 0), COUNT(DISTINCT CASE WHEN u > 0 THEN a END), "
|
| 303 |
f"(SELECT COUNT(*) FROM r WHERE a IS NOT NULL) - COUNT(*), (SELECT COALESCE(SUM(s), 0) FROM r), "
|
| 304 |
+
f"(SELECT COALESCE(SUM(u), 0) FROM r), "
|
| 305 |
+
f"(SELECT COUNT(*) FROM (SELECT a FROM kept GROUP BY a HAVING COUNT(*) > 1)) FROM kept").fetchone()
|
| 306 |
return {"sessions": round(float(sessions), 4), "units": round(float(units), 4), "asinsWithUnits": int(count),
|
| 307 |
"supersededRows": int(superseded), "rawSessions": round(float(raw_sessions), 4),
|
| 308 |
+
"rawUnits": round(float(raw_units), 4), "summedAsins": int(summed)}
|
| 309 |
|
| 310 |
|
| 311 |
def _rank(raw):
|
|
|
|
| 737 |
try:
|
| 738 |
def _note(doc):
|
| 739 |
doc = doc if isinstance(doc, dict) else {}
|
| 740 |
+
doc["last_error"] = (READ_FAILED.format(cause=exc) if isinstance(exc, _ReadFailed) else
|
| 741 |
+
f"The weekly history could not be written ({type(exc).__name__}): {exc}")
|
| 742 |
return doc
|
| 743 |
_rt(tenant_key, st).update(KEY_INDEX, _note)
|
| 744 |
except Exception: # noqa: BLE001
|
|
|
|
| 904 |
if not chosen:
|
| 905 |
return {"ok": False, "app_total": None, "independent_total": None, "weeks": [],
|
| 906 |
"detail": "No weekly history is held for that period, so nothing can be tied."}
|
| 907 |
+
live, failed = None, None
|
| 908 |
if held and held[-1] in chosen:
|
| 909 |
+
try:
|
| 910 |
+
with _reader(tenant_key, st) as cur:
|
| 911 |
+
live = source_totals(cur) if cur is not None else None
|
| 912 |
+
except _ReadFailed as exc:
|
| 913 |
+
failed = (f"The Amazon mirror could not be read just now ({exc}), so the newest week cannot be "
|
| 914 |
+
"tied. Try again in a minute.")
|
| 915 |
rows, ok = [], True
|
| 916 |
app = {"sessions": 0.0, "units": 0.0}
|
| 917 |
ind = {"sessions": 0.0, "units": 0.0}
|
|
|
|
| 933 |
bad = [r["week"] for r in rows if not r["ok"]]
|
| 934 |
detail = (f"{len(rows)} weeks tie sessions and units to the Sales and Traffic totals." if ok else
|
| 935 |
"These weeks do not tie to the Sales and Traffic totals: " + ", ".join(bad) + ".")
|
| 936 |
+
if failed:
|
| 937 |
+
detail += " " + failed
|
| 938 |
superseded = int((live or {}).get("supersededRows") or 0)
|
| 939 |
if superseded:
|
| 940 |
detail += (f" {superseded} older Sales and Traffic rows are left out because a newer report covers "
|
platform/modules/restock_planner.py
CHANGED
|
@@ -37,6 +37,9 @@ REOPEN_FROM = ("Watching", "Done", "Dismissed")
|
|
| 37 |
DECISIONS = {"approved": "Accepted", "dismissed": "Dismissed"}
|
| 38 |
DEFAULTS = {"send_to_fba_days": 30.0, "overstock_days": 90.0, "due_days": 7}
|
| 39 |
NOT_READY = "The restock plan starts after the next Amazon inventory sync."
|
|
|
|
|
|
|
|
|
|
| 40 |
LIMIT = ("Each restock planner document is limited to 128 MB by the store. About 60,000 SKUs fit in "
|
| 41 |
"the items document, and history splits by month. Split the items document by band when "
|
| 42 |
"it nears that size.")
|
|
@@ -106,9 +109,15 @@ def _mirror_path(tenant_key, st):
|
|
| 106 |
return amazon_store._tenant_mirror_path(tenant_key, rt)
|
| 107 |
|
| 108 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 109 |
@contextlib.contextmanager
|
| 110 |
def _reader(tenant_key, st):
|
| 111 |
-
"""A read-only cursor on the tenant mirror,
|
|
|
|
|
|
|
| 112 |
path = Path(_mirror_path(tenant_key, st))
|
| 113 |
if not path.exists():
|
| 114 |
yield None
|
|
@@ -119,29 +128,29 @@ def _reader(tenant_key, st):
|
|
| 119 |
except Exception: # noqa: BLE001
|
| 120 |
shared = False
|
| 121 |
# The process holds one writer on its own mirror: a separate read-only connect would refuse.
|
| 122 |
-
if shared:
|
| 123 |
-
cur = datastore.ro_cursor()
|
| 124 |
-
try:
|
| 125 |
-
yield cur
|
| 126 |
-
finally:
|
| 127 |
-
cur.close()
|
| 128 |
-
return
|
| 129 |
-
import duckdb
|
| 130 |
-
con = duckdb.connect(str(path), read_only=True)
|
| 131 |
try:
|
| 132 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 133 |
finally:
|
| 134 |
-
|
| 135 |
|
| 136 |
|
| 137 |
def _columns(cur, table):
|
|
|
|
| 138 |
if cur is None:
|
| 139 |
return set()
|
| 140 |
try:
|
| 141 |
return {r[0] for r in cur.execute(
|
| 142 |
"SELECT column_name FROM information_schema.columns WHERE table_name = ?", [table]).fetchall()}
|
| 143 |
-
except Exception:
|
| 144 |
-
|
| 145 |
|
| 146 |
|
| 147 |
def _newest(cols, table):
|
|
@@ -466,11 +475,22 @@ def refresh(tenant_key, *, st=None, now=None):
|
|
| 466 |
now = now or _now()
|
| 467 |
conf = settings(tenant_key, st=st)
|
| 468 |
links = _links(st)
|
| 469 |
-
|
| 470 |
-
|
| 471 |
-
|
| 472 |
-
|
| 473 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 474 |
if rows is None:
|
| 475 |
message = notices[0] if notices else NOT_READY
|
| 476 |
|
|
@@ -734,15 +754,20 @@ def validate(tenant_key, *, st=None, **period):
|
|
| 734 |
"""Rule 8: FBA available over the planner's rows ties to the planning table's own total."""
|
| 735 |
st = _rt(tenant_key, st)
|
| 736 |
conf = settings(tenant_key, st=st)
|
| 737 |
-
|
| 738 |
-
|
| 739 |
-
|
| 740 |
-
|
| 741 |
-
|
| 742 |
-
|
| 743 |
-
|
| 744 |
-
|
| 745 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 746 |
planned = [r for r in rows or () if r["availableSource"] == "planning"]
|
| 747 |
app_total = float(sum(r["fbaAvailable"] or 0 for r in planned))
|
| 748 |
independent_total = float(independent[0] or 0)
|
|
|
|
| 37 |
DECISIONS = {"approved": "Accepted", "dismissed": "Dismissed"}
|
| 38 |
DEFAULTS = {"send_to_fba_days": 30.0, "overstock_days": 90.0, "due_days": 7}
|
| 39 |
NOT_READY = "The restock plan starts after the next Amazon inventory sync."
|
| 40 |
+
#: The mirror holds the tables but this read failed or was busy. `{cause}` is the error class.
|
| 41 |
+
READ_FAILED = ("The Amazon mirror could not be read just now ({cause}), so the restock plan was not refreshed. "
|
| 42 |
+
"The next Amazon sync reads it again.")
|
| 43 |
LIMIT = ("Each restock planner document is limited to 128 MB by the store. About 60,000 SKUs fit in "
|
| 44 |
"the items document, and history splits by month. Split the items document by band when "
|
| 45 |
"it nears that size.")
|
|
|
|
| 109 |
return amazon_store._tenant_mirror_path(tenant_key, rt)
|
| 110 |
|
| 111 |
|
| 112 |
+
class _ReadFailed(Exception):
|
| 113 |
+
"""A mirror read failed, was locked or busy. Never read as an absent table."""
|
| 114 |
+
|
| 115 |
+
|
| 116 |
@contextlib.contextmanager
|
| 117 |
def _reader(tenant_key, st):
|
| 118 |
+
"""A read-only cursor on the tenant mirror, None when the mirror file does not exist.
|
| 119 |
+
|
| 120 |
+
An open failure raises `_ReadFailed` naming its error class."""
|
| 121 |
path = Path(_mirror_path(tenant_key, st))
|
| 122 |
if not path.exists():
|
| 123 |
yield None
|
|
|
|
| 128 |
except Exception: # noqa: BLE001
|
| 129 |
shared = False
|
| 130 |
# The process holds one writer on its own mirror: a separate read-only connect would refuse.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 131 |
try:
|
| 132 |
+
if shared:
|
| 133 |
+
cur = datastore.ro_cursor()
|
| 134 |
+
else:
|
| 135 |
+
import duckdb
|
| 136 |
+
cur = duckdb.connect(str(path), read_only=True)
|
| 137 |
+
except Exception as exc: # noqa: BLE001
|
| 138 |
+
raise _ReadFailed(type(exc).__name__) from exc
|
| 139 |
+
try:
|
| 140 |
+
yield cur
|
| 141 |
finally:
|
| 142 |
+
cur.close()
|
| 143 |
|
| 144 |
|
| 145 |
def _columns(cur, table):
|
| 146 |
+
"""The table's column names, an empty set when it is absent. A failed read raises `_ReadFailed`."""
|
| 147 |
if cur is None:
|
| 148 |
return set()
|
| 149 |
try:
|
| 150 |
return {r[0] for r in cur.execute(
|
| 151 |
"SELECT column_name FROM information_schema.columns WHERE table_name = ?", [table]).fetchall()}
|
| 152 |
+
except Exception as exc: # noqa: BLE001
|
| 153 |
+
raise _ReadFailed(type(exc).__name__) from exc
|
| 154 |
|
| 155 |
|
| 156 |
def _newest(cols, table):
|
|
|
|
| 475 |
now = now or _now()
|
| 476 |
conf = settings(tenant_key, st=st)
|
| 477 |
links = _links(st)
|
| 478 |
+
try:
|
| 479 |
+
with _reader(tenant_key, st) as cur:
|
| 480 |
+
if cur is None:
|
| 481 |
+
rows, notices = None, ["The tenant mirror does not exist yet."]
|
| 482 |
+
else:
|
| 483 |
+
rows, notices = build_rows(cur, links, conf)
|
| 484 |
+
except _ReadFailed as exc:
|
| 485 |
+
# The held plan stays as it was: a failed read is never "not synced" and never an empty plan.
|
| 486 |
+
failed = READ_FAILED.format(cause=exc)
|
| 487 |
+
|
| 488 |
+
def _failed(doc):
|
| 489 |
+
doc = doc if isinstance(doc, dict) else {}
|
| 490 |
+
doc.update(last_error=failed, checkedAt=_stamp(now))
|
| 491 |
+
return doc
|
| 492 |
+
st.update(KEY_ITEMS, _failed)
|
| 493 |
+
return {"state": "read_failed", "message": failed}
|
| 494 |
if rows is None:
|
| 495 |
message = notices[0] if notices else NOT_READY
|
| 496 |
|
|
|
|
| 754 |
"""Rule 8: FBA available over the planner's rows ties to the planning table's own total."""
|
| 755 |
st = _rt(tenant_key, st)
|
| 756 |
conf = settings(tenant_key, st=st)
|
| 757 |
+
try:
|
| 758 |
+
with _reader(tenant_key, st) as cur:
|
| 759 |
+
if cur is None or "available" not in _columns(cur, PLANNING):
|
| 760 |
+
return {"ok": False, "app_total": None, "independent_total": None,
|
| 761 |
+
"detail": f"{SOURCE_NAMES[PLANNING]} has not synced yet."}
|
| 762 |
+
rows, _notices = build_rows(cur, _links(st), conf)
|
| 763 |
+
cols = _columns(cur, PLANNING)
|
| 764 |
+
independent = cur.execute(
|
| 765 |
+
f'SELECT SUM(TRY_CAST("available" AS DOUBLE)), COUNT(DISTINCT "sku") FROM {PLANNING} '
|
| 766 |
+
f'WHERE {_newest(cols, PLANNING)}').fetchone()
|
| 767 |
+
except _ReadFailed as exc:
|
| 768 |
+
return {"ok": False, "app_total": None, "independent_total": None,
|
| 769 |
+
"detail": f"The Amazon mirror could not be read just now ({exc}), so nothing can be tied. "
|
| 770 |
+
"Try again in a minute."}
|
| 771 |
planned = [r for r in rows or () if r["availableSource"] == "planning"]
|
| 772 |
app_total = float(sum(r["fbaAvailable"] or 0 for r in planned))
|
| 773 |
independent_total = float(independent[0] or 0)
|
web/public/AIOS_DEPLOY_RECEIPT
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
|
|
|
|
| 1 |
+
0ae8b932683b55f2b15d76ac37e314ae821910fcbbc157e63bc0fe6cacf75cfe
|