Deploy AIOS web (React glide grid + FastAPI slice)
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- RELEASES.json +1 -1
- VERSION +1 -1
- api/amazon_quotes_loop.py +5 -93
- api/amazon_relational.py +1 -6
- api/app_templates.py +1 -11
- api/application_registry.py +6 -28
- api/connectors_amazon.py +10 -5
- api/external_freshness.py +55 -150
- api/main.py +26 -52
- api/odoo_relational.py +7 -1
- api/process_role.py +35 -101
- api/routes_amazon_projects.py +3 -9
- api/routes_brand.py +2 -52
- api/routes_changes.py +0 -3
- api/routes_competitors.py +10 -32
- api/routes_connectors.py +6 -1
- api/routes_grid.py +0 -7
- api/routes_odoo_tables.py +2 -10
- api/routes_shares.py +32 -27
- api/routes_starred.py +0 -5
- api/routes_tables.py +3 -25
- api/sync_runtime.py +10 -55
- api/test_odoo_rank_refresh.py +70 -1
- api/usage_ledger.py +53 -59
- platform/core/amazon_projects.py +21 -90
- platform/core/amazon_rivals.py +117 -575
- platform/core/connector_spec.py +1 -46
- platform/core/external_tools.py +1 -1
- platform/core/shares.py +2 -24
- platform/harness/amazon_client.py +2 -5
- platform/harness/amazon_schema.py +13 -22
- platform/harness/amazon_store.py +0 -0
- platform/harness/datastore.py +34 -44
- platform/harness/mirror_store.py +152 -394
- platform/modules/amazon_profit.py +23 -207
- platform/modules/amazon_profit_account.py +7 -122
- platform/modules/competitor_watch.py +3 -19
- requirements.txt +0 -1
- web/package-lock.json +0 -485
- web/package-lock.linux.sha256 +2 -2
- web/package.json +0 -1
- web/public/AIOS_DEPLOY_RECEIPT +1 -1
- web/src/apps/amazon-profit/AppView.tsx +6 -81
- web/src/apps/amazon-profit/ProjectOverview.tsx +9 -32
- web/src/apps/amazon-profit/ProjectSkuPicker.tsx +14 -43
- web/src/apps/amazon-profit/Projects.tsx +20 -53
- web/src/apps/amazon-profit/ProjectsShared.tsx +3 -20
- web/src/apps/templateMarks.tsx +0 -4
- web/src/automation/AutomationDetail.tsx +23 -1
- web/src/automation/AutomationSurface.tsx +22 -2
RELEASES.json
CHANGED
|
@@ -1,4 +1,4 @@
|
|
| 1 |
{
|
| 2 |
-
"current": "
|
| 3 |
"releases": []
|
| 4 |
}
|
|
|
|
| 1 |
{
|
| 2 |
+
"current": "b1a532ae668",
|
| 3 |
"releases": []
|
| 4 |
}
|
VERSION
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
|
|
|
|
| 1 |
+
b1a532ae668
|
api/amazon_quotes_loop.py
CHANGED
|
@@ -9,12 +9,10 @@ import logging
|
|
| 9 |
from pathlib import Path
|
| 10 |
import threading
|
| 11 |
import time
|
| 12 |
-
import uuid
|
| 13 |
|
| 14 |
JOB = "amazon-rival-quotes"
|
| 15 |
#: The loop asks each tenant whether its daily run is due this often.
|
| 16 |
POLL_SECONDS = 600
|
| 17 |
-
SNAPSHOT_TABLES = ("competitor_snapshots", "competitor_images")
|
| 18 |
|
| 19 |
#: Called as `fn(tenant, changed_asins)` after a run finishes, where `changed_asins` is a tuple of the
|
| 20 |
#: ASINs whose newest quote changed. Each import is lazy and a failure is one log line, never a failed run.
|
|
@@ -28,29 +26,17 @@ _thread = None
|
|
| 28 |
_stop = None
|
| 29 |
|
| 30 |
|
| 31 |
-
class SnapshotBlocked(RuntimeError):
|
| 32 |
-
"""A safe admission cause that can be recorded without exposing provider details."""
|
| 33 |
-
|
| 34 |
-
|
| 35 |
def _say(message):
|
| 36 |
print(f"[{JOB}] {message}", flush=True)
|
| 37 |
|
| 38 |
|
| 39 |
def readers(fields):
|
| 40 |
-
"""
|
| 41 |
from harness import amazon_client as cx
|
| 42 |
-
from harness import amazon_competitor_facts as facts
|
| 43 |
-
|
| 44 |
-
def complete(asins):
|
| 45 |
-
catalog, offers = facts.read_facts(fields, asins), facts.read_offers(fields, asins)
|
| 46 |
-
return {asin: {**catalog[asin], **{key: offers[asin][key] for key in ("prime", "offers")},
|
| 47 |
-
"cause": " ".join(c for c in (catalog[asin]["cause"], offers[asin]["cause"]) if c) or None}
|
| 48 |
-
for asin in asins}
|
| 49 |
-
|
| 50 |
return {"offers": lambda asins: cx.item_offers(fields, asins),
|
| 51 |
# The pricing sync draws this same `competitive_summary` pacer in this worker process.
|
| 52 |
"summaries": lambda asins: cx.competitive_summaries(fields, asins),
|
| 53 |
-
"
|
| 54 |
|
| 55 |
|
| 56 |
@contextmanager
|
|
@@ -78,69 +64,9 @@ def _after_quotes(tenant, changed):
|
|
| 78 |
"the run result is unchanged.")
|
| 79 |
|
| 80 |
|
| 81 |
-
def _snapshot_cause(runtime, *, check_lease=True):
|
| 82 |
-
from harness import amazon_store
|
| 83 |
-
|
| 84 |
-
enabled = amazon_store.read_database_enablement(runtime)
|
| 85 |
-
if not enabled["available"]:
|
| 86 |
-
return "Competitor database settings could not be read. Restore settings access and retry the run."
|
| 87 |
-
if not all(enabled["databases"].get("ut_amzn_" + table) for table in SNAPSHOT_TABLES):
|
| 88 |
-
return "A competitor database is disabled. Enable competitor snapshots and images in Settings, Connectors and retry."
|
| 89 |
-
if check_lease and amazon_store.first_load_job_live(runtime):
|
| 90 |
-
return "The Amazon mirror writer lease is held. Let the current writer finish and retry the run."
|
| 91 |
-
return None
|
| 92 |
-
|
| 93 |
-
|
| 94 |
-
def _publish_snapshots(tenant, runtime, result):
|
| 95 |
-
from harness import amazon_competitor_facts as facts, amazon_store, mirror_store
|
| 96 |
-
|
| 97 |
-
lock = mirror_store.for_connector("amazon").lock
|
| 98 |
-
if not lock.acquire(blocking=False):
|
| 99 |
-
raise SnapshotBlocked("The Amazon mirror writer is busy.")
|
| 100 |
-
holder = f"{JOB}:{result['id']}"
|
| 101 |
-
leased = False
|
| 102 |
-
try:
|
| 103 |
-
leased = amazon_store.acquire_amazon_lease(runtime, holder)
|
| 104 |
-
if not leased:
|
| 105 |
-
raise SnapshotBlocked("The Amazon mirror writer lease is held.")
|
| 106 |
-
cause = _snapshot_cause(runtime, check_lease=False)
|
| 107 |
-
if cause:
|
| 108 |
-
raise SnapshotBlocked(cause)
|
| 109 |
-
report = {"tables": {}, "problems": []}
|
| 110 |
-
with mirror_store.sync_mirror("amazon", tenant, runtime, report=report, expected_tables=SNAPSHOT_TABLES):
|
| 111 |
-
con, _path = amazon_store._connect_tenant_mirror(tenant, runtime)
|
| 112 |
-
try:
|
| 113 |
-
stored = facts.import_snapshots(con, result["snapshots"])
|
| 114 |
-
for table in SNAPSHOT_TABLES:
|
| 115 |
-
total = con.execute(f"SELECT count(*) FROM amzn_{table}").fetchone()[0]
|
| 116 |
-
report["tables"][table] = {"state": "done", "rows": total}
|
| 117 |
-
finally:
|
| 118 |
-
con.close()
|
| 119 |
-
if report["problems"]:
|
| 120 |
-
raise RuntimeError("Competitor mirror publication failed")
|
| 121 |
-
result["counts"].update(snapshots=stored["snapshots"], images=stored["images"])
|
| 122 |
-
finally:
|
| 123 |
-
try:
|
| 124 |
-
if leased:
|
| 125 |
-
amazon_store.release_amazon_lease(runtime, holder)
|
| 126 |
-
finally:
|
| 127 |
-
lock.release()
|
| 128 |
-
|
| 129 |
-
|
| 130 |
-
def _run_blocked(runtime, cause):
|
| 131 |
-
from core import amazon_rivals as rivals
|
| 132 |
-
|
| 133 |
-
run_id = uuid.uuid4().hex
|
| 134 |
-
rivals.start_run(runtime, run_id, asin_total=0)
|
| 135 |
-
return {"id": run_id, "progress": 0, "cause": cause,
|
| 136 |
-
"counts": {"facts": dict.fromkeys(rivals.FACT_COUNT_COLUMNS, 0)}, "changed": (), "snapshots": []}
|
| 137 |
-
|
| 138 |
-
|
| 139 |
def run_tenant(tenant, runtime, *, now=None, fields=None, sleep=time.sleep):
|
| 140 |
"""Run today's quotes for one tenant when due. None when it is not due or has no Amazon store."""
|
| 141 |
from core import amazon_rivals as rivals
|
| 142 |
-
if str(tenant) != str(runtime.key) or str(runtime.key) != str(runtime.tenant.key):
|
| 143 |
-
raise ValueError("Competitor quotes require the matching tenant runtime")
|
| 144 |
try:
|
| 145 |
if not rivals.due(runtime, now=now):
|
| 146 |
return None
|
|
@@ -154,23 +80,9 @@ def run_tenant(tenant, runtime, *, now=None, fields=None, sleep=time.sleep):
|
|
| 154 |
fields = amazon_store.credential(tenant, runtime)
|
| 155 |
if not fields:
|
| 156 |
return None
|
| 157 |
-
|
| 158 |
-
|
| 159 |
-
|
| 160 |
-
rivals.run_quotes(runtime, lambda: _mirror(runtime), **readers(fields),
|
| 161 |
-
seller_id=str(fields.get("seller_id") or "").strip() or None, now=now,
|
| 162 |
-
log=lambda message: _say(f"tenant={tenant} {message}"), sleep=sleep))
|
| 163 |
-
if result["snapshots"]:
|
| 164 |
-
try:
|
| 165 |
-
_publish_snapshots(tenant, runtime, result)
|
| 166 |
-
except Exception as exc:
|
| 167 |
-
cause = (str(exc) if isinstance(exc, SnapshotBlocked) else
|
| 168 |
-
f"Competitor snapshots could not be published ({type(exc).__name__}).")
|
| 169 |
-
result["cause"] = (f"{cause} Check database settings and mirror access, let any writer finish, "
|
| 170 |
-
f"and retry within {rivals.RETRY_RUN_HOURS} hours.")
|
| 171 |
-
result["counts"]["elapsed_seconds"] = round(time.monotonic() - started, 3)
|
| 172 |
-
rivals.mark_run(runtime, result["id"], progress=result["progress"], finished=True,
|
| 173 |
-
counts=result["counts"], cause=result["cause"])
|
| 174 |
if result["cause"] is None:
|
| 175 |
_after_quotes(tenant, result["changed"])
|
| 176 |
return result
|
|
|
|
| 9 |
from pathlib import Path
|
| 10 |
import threading
|
| 11 |
import time
|
|
|
|
| 12 |
|
| 13 |
JOB = "amazon-rival-quotes"
|
| 14 |
#: The loop asks each tenant whether its daily run is due this often.
|
| 15 |
POLL_SECONDS = 600
|
|
|
|
| 16 |
|
| 17 |
#: Called as `fn(tenant, changed_asins)` after a run finishes, where `changed_asins` is a tuple of the
|
| 18 |
#: ASINs whose newest quote changed. Each import is lazy and a failure is one log line, never a failed run.
|
|
|
|
| 26 |
_stop = None
|
| 27 |
|
| 28 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 29 |
def _say(message):
|
| 30 |
print(f"[{JOB}] {message}", flush=True)
|
| 31 |
|
| 32 |
|
| 33 |
def readers(fields):
|
| 34 |
+
"""The three SP-API reads of one tenant's credential. Each takes a list of at most 20 ASINs."""
|
| 35 |
from harness import amazon_client as cx
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 36 |
return {"offers": lambda asins: cx.item_offers(fields, asins),
|
| 37 |
# The pricing sync draws this same `competitive_summary` pacer in this worker process.
|
| 38 |
"summaries": lambda asins: cx.competitive_summaries(fields, asins),
|
| 39 |
+
"ranks": lambda asins: cx.catalog_sales_ranks(fields, asins)}
|
| 40 |
|
| 41 |
|
| 42 |
@contextmanager
|
|
|
|
| 64 |
"the run result is unchanged.")
|
| 65 |
|
| 66 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 67 |
def run_tenant(tenant, runtime, *, now=None, fields=None, sleep=time.sleep):
|
| 68 |
"""Run today's quotes for one tenant when due. None when it is not due or has no Amazon store."""
|
| 69 |
from core import amazon_rivals as rivals
|
|
|
|
|
|
|
| 70 |
try:
|
| 71 |
if not rivals.due(runtime, now=now):
|
| 72 |
return None
|
|
|
|
| 80 |
fields = amazon_store.credential(tenant, runtime)
|
| 81 |
if not fields:
|
| 82 |
return None
|
| 83 |
+
result = rivals.run_quotes(runtime, lambda: _mirror(runtime), **readers(fields),
|
| 84 |
+
seller_id=str(fields.get("seller_id") or "").strip() or None, now=now,
|
| 85 |
+
log=lambda message: _say(f"tenant={tenant} {message}"), sleep=sleep)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 86 |
if result["cause"] is None:
|
| 87 |
_after_quotes(tenant, result["changed"])
|
| 88 |
return result
|
api/amazon_relational.py
CHANGED
|
@@ -650,8 +650,6 @@ def _fields_for(cur, key, *, read_through=False):
|
|
| 650 |
"pinned": name == pinned,
|
| 651 |
**({"description": amazon_schema.REVIEW_UNAVAILABLE_NOTE}
|
| 652 |
if name == "asin" and key in (PREFIX + "review_topics", PREFIX + "review_trends") else {}),
|
| 653 |
-
**({"description": amazon_schema.SQP_LIMIT_NOTE}
|
| 654 |
-
if name == "asin" and key == PREFIX + "search_query_performance" else {}),
|
| 655 |
**({"format": fmt} if fmt else {}),
|
| 656 |
}, flow="amazon_relational"))
|
| 657 |
for lookup in derived.get(name, ()):
|
|
@@ -725,8 +723,7 @@ READ_THROUGH = frozenset(
|
|
| 725 |
and (spec.get("report") or spec.get("api") or spec.get("monthly")))) | frozenset(
|
| 726 |
PREFIX + table for table in amazon_store.DERIVED_TABLES) | frozenset(
|
| 727 |
# Change history grows without a cap (standing rule 1), so it is never stored.
|
| 728 |
-
PREFIX + table for table in amazon_store.HISTORY_TABLES)
|
| 729 |
-
PREFIX + "competitor_snapshots", PREFIX + "competitor_images"))
|
| 730 |
|
| 731 |
|
| 732 |
def _quoted(name):
|
|
@@ -1601,8 +1598,6 @@ def apply_plan(rt, built, username="amazon", today=None):
|
|
| 1601 |
doc[key]["description"] = ads_descriptions[key]
|
| 1602 |
if key in (PREFIX + "review_topics", PREFIX + "review_trends"):
|
| 1603 |
doc[key]["description"] = amazon_schema.REVIEW_UNAVAILABLE_NOTE
|
| 1604 |
-
if key == PREFIX + "search_query_performance":
|
| 1605 |
-
doc[key]["description"] = amazon_schema.SQP_LIMIT_NOTE
|
| 1606 |
if key == PREFIX + "listing_items":
|
| 1607 |
doc[key]["amazonListingIdentity"] = listing_version
|
| 1608 |
return doc
|
|
|
|
| 650 |
"pinned": name == pinned,
|
| 651 |
**({"description": amazon_schema.REVIEW_UNAVAILABLE_NOTE}
|
| 652 |
if name == "asin" and key in (PREFIX + "review_topics", PREFIX + "review_trends") else {}),
|
|
|
|
|
|
|
| 653 |
**({"format": fmt} if fmt else {}),
|
| 654 |
}, flow="amazon_relational"))
|
| 655 |
for lookup in derived.get(name, ()):
|
|
|
|
| 723 |
and (spec.get("report") or spec.get("api") or spec.get("monthly")))) | frozenset(
|
| 724 |
PREFIX + table for table in amazon_store.DERIVED_TABLES) | frozenset(
|
| 725 |
# Change history grows without a cap (standing rule 1), so it is never stored.
|
| 726 |
+
PREFIX + table for table in amazon_store.HISTORY_TABLES)
|
|
|
|
| 727 |
|
| 728 |
|
| 729 |
def _quoted(name):
|
|
|
|
| 1598 |
doc[key]["description"] = ads_descriptions[key]
|
| 1599 |
if key in (PREFIX + "review_topics", PREFIX + "review_trends"):
|
| 1600 |
doc[key]["description"] = amazon_schema.REVIEW_UNAVAILABLE_NOTE
|
|
|
|
|
|
|
| 1601 |
if key == PREFIX + "listing_items":
|
| 1602 |
doc[key]["amazonListingIdentity"] = listing_version
|
| 1603 |
return doc
|
api/app_templates.py
CHANGED
|
@@ -127,22 +127,12 @@ TEMPLATES = (
|
|
| 127 |
# Project management reads its own module tables (ut_amazon_*), so it declares no
|
| 128 |
# source: a cold mirror must not refuse an App whose rows live in the store.
|
| 129 |
_amazon("amazon_profit", "Project management", "See profit, ads and stock for every SKU.",
|
| 130 |
-
"Create value", (),
|
| 131 |
-
optional_sources=("ut_amzn_search_query_performance", "ut_amzn_order_items",
|
| 132 |
-
"ut_amzn_brand_analytics", "ut_amzn_listings", "ut_amzn_catalog_items",
|
| 133 |
-
"ut_amzn_listing_items", "ut_amzn_competitor_snapshots",
|
| 134 |
-
"ut_amzn_competitor_images"),
|
| 135 |
-
optional_connectors=("odoo",), proposes=False),
|
| 136 |
{"kind": "alerts", "label": "Alerts", "purpose": "Act on changes in your databases.",
|
| 137 |
"connector": None, "connectors": (), "permission": "automation", "tags": ["Automation"],
|
| 138 |
"sources": (), "optional_sources": (), "optional_connectors": (), "process": "None",
|
| 139 |
"mark": "alerts", "module": None, "router": "routes_automation", "view": "alerts",
|
| 140 |
"tools": (), "proposes": False},
|
| 141 |
-
{"kind": "reports", "label": "Reports", "purpose": "Download reports shared with you.",
|
| 142 |
-
"connector": "odoo", "connectors": ("odoo",), "tags": ["Odoo"], "sources": (),
|
| 143 |
-
"optional_sources": (), "optional_connectors": (), "process": "None", "mark": "reports",
|
| 144 |
-
"module": "modules.reports", "router": "routes_reports", "view": "reports",
|
| 145 |
-
"tools": (), "proposes": False},
|
| 146 |
)
|
| 147 |
if not all(validate_connectors(row) for row in TEMPLATES):
|
| 148 |
raise ValueError("An App template omits a source connector.")
|
|
|
|
| 127 |
# Project management reads its own module tables (ut_amazon_*), so it declares no
|
| 128 |
# source: a cold mirror must not refuse an App whose rows live in the store.
|
| 129 |
_amazon("amazon_profit", "Project management", "See profit, ads and stock for every SKU.",
|
| 130 |
+
"Create value", (), optional_connectors=("odoo",), proposes=False),
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 131 |
{"kind": "alerts", "label": "Alerts", "purpose": "Act on changes in your databases.",
|
| 132 |
"connector": None, "connectors": (), "permission": "automation", "tags": ["Automation"],
|
| 133 |
"sources": (), "optional_sources": (), "optional_connectors": (), "process": "None",
|
| 134 |
"mark": "alerts", "module": None, "router": "routes_automation", "view": "alerts",
|
| 135 |
"tools": (), "proposes": False},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 136 |
)
|
| 137 |
if not all(validate_connectors(row) for row in TEMPLATES):
|
| 138 |
raise ValueError("An App template omits a source connector.")
|
api/application_registry.py
CHANGED
|
@@ -624,16 +624,6 @@ def _source_rows_exist(session, sources):
|
|
| 624 |
return False
|
| 625 |
|
| 626 |
|
| 627 |
-
def _reports_readable(runtime, user, is_admin):
|
| 628 |
-
"""Report grants admit the Reports App independently of its App sharing record."""
|
| 629 |
-
try:
|
| 630 |
-
from modules import reports
|
| 631 |
-
except Exception as exc: # A failed Reports import withholds its App.
|
| 632 |
-
raise_if_resource_error(exc)
|
| 633 |
-
return False
|
| 634 |
-
return bool(is_admin or reports.granted_keys(runtime, user))
|
| 635 |
-
|
| 636 |
-
|
| 637 |
def _readable_template_app(session, app_id):
|
| 638 |
"""Resolve App identity before reading any source or exposing a refusal."""
|
| 639 |
from deps import err
|
|
@@ -645,10 +635,7 @@ def _readable_template_app(session, app_id):
|
|
| 645 |
except Exception:
|
| 646 |
raise err(503, "app_unavailable", "This App cannot be read right now.") from None
|
| 647 |
if (not isinstance(app, dict) or app.get("id") != app_id or app.get("tenant") != session.tenant
|
| 648 |
-
or app.get("retired") is True
|
| 649 |
-
or not (_reports_readable(session.runtime, session.uname, session.admin)
|
| 650 |
-
if app.get("templateKind") == "reports"
|
| 651 |
-
else may_read(app, session.uname, session.admin))):
|
| 652 |
raise err(404, "app_not_found", "This App is unavailable or is not shared with you.")
|
| 653 |
kind = app.get("templateKind")
|
| 654 |
catalogue = _app_templates.BY_KIND.get(str(kind or ""))
|
|
@@ -1318,7 +1305,7 @@ def validate_application(payload, tenant, *, semantic_records=None, field_contra
|
|
| 1318 |
|
| 1319 |
|
| 1320 |
def _public(app, user, is_admin=False, *, allowed_databases=None, target_database=None,
|
| 1321 |
-
is_default=False
|
| 1322 |
row=copy.deepcopy(_app_projection(app))
|
| 1323 |
row.setdefault("origin", None)
|
| 1324 |
if row.get("type") == "code":
|
|
@@ -1327,10 +1314,9 @@ def _public(app, user, is_admin=False, *, allowed_databases=None, target_databas
|
|
| 1327 |
compatible = [database for database in row.get("compatibleDatabases") or ()
|
| 1328 |
if database in _allowed(row.get("compatibleDatabases"), allowed_databases)]
|
| 1329 |
row["compatibleDatabases"] = compatible
|
| 1330 |
-
|
| 1331 |
-
row["capabilities"]={"read":app_readable if readable is None else readable,"edit":may_edit(app,user,is_admin),
|
| 1332 |
"share":may_edit(app,user,is_admin),"delete":may_edit(app,user,is_admin),
|
| 1333 |
-
"duplicate":
|
| 1334 |
"default":bool(is_admin)}
|
| 1335 |
row["starred"] = user in (app.get("starredBy") or ())
|
| 1336 |
row["isDefault"] = bool(is_default)
|
|
@@ -1391,7 +1377,6 @@ def list_apps(runtime, user, is_admin=False, *, database=None, kind=None, starre
|
|
| 1391 |
"""List Applications. Unfiltered is the App home; a database filter is how a View resolves."""
|
| 1392 |
doc = load(runtime)
|
| 1393 |
rows=[]
|
| 1394 |
-
reports_reader = None
|
| 1395 |
for app in doc["applications"].values():
|
| 1396 |
if (isinstance(app, dict) and getattr(runtime, "key", None)
|
| 1397 |
and app.get("tenant") != runtime.key):
|
|
@@ -1401,19 +1386,13 @@ def list_apps(runtime, user, is_admin=False, *, database=None, kind=None, starre
|
|
| 1401 |
if (isinstance(app, dict) and app.get("templateKind") == "alerts"
|
| 1402 |
and not (may_open or _user_wall(runtime, user))("automation")):
|
| 1403 |
continue
|
| 1404 |
-
report_app = isinstance(app, dict) and app.get("templateKind") == "reports"
|
| 1405 |
-
if report_app:
|
| 1406 |
-
if reports_reader is None:
|
| 1407 |
-
reports_reader = _reports_readable(runtime, user, is_admin)
|
| 1408 |
-
if not reports_reader or app.get("retired") is True:
|
| 1409 |
-
continue
|
| 1410 |
if isinstance(app, dict) and app.get("templateKind") in ("customer_card", FINANCE_TEMPLATE_KIND):
|
| 1411 |
continue
|
| 1412 |
if database:
|
| 1413 |
# A View resolves its App only through this read, so a retired seed stays listed here.
|
| 1414 |
-
if not
|
| 1415 |
if database not in (app.get("compatibleDatabases") or ()): continue
|
| 1416 |
-
elif not
|
| 1417 |
if (not _database_free(app)
|
| 1418 |
and not _allowed(app.get("compatibleDatabases"), allowed_databases)):
|
| 1419 |
catalogue = _app_templates.BY_KIND.get(str(app.get("templateKind") or ""))
|
|
@@ -1425,7 +1404,6 @@ def list_apps(runtime, user, is_admin=False, *, database=None, kind=None, starre
|
|
| 1425 |
default_key = f"{database}::{app.get('type')}" if database else ""
|
| 1426 |
rows.append(_public(app,user,is_admin,allowed_databases=allowed_databases,
|
| 1427 |
target_database=database,
|
| 1428 |
-
readable=True if report_app else None,
|
| 1429 |
is_default=bool(default_key and doc["defaults"].get(default_key)==app.get("id"))))
|
| 1430 |
# S67-T39: the dependent-View count rides a DATABASE-filtered read only, and only for rows
|
| 1431 |
# the caller may delete, so the unfiltered nav read (App home tiles) pays nothing for it.
|
|
|
|
| 624 |
return False
|
| 625 |
|
| 626 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 627 |
def _readable_template_app(session, app_id):
|
| 628 |
"""Resolve App identity before reading any source or exposing a refusal."""
|
| 629 |
from deps import err
|
|
|
|
| 635 |
except Exception:
|
| 636 |
raise err(503, "app_unavailable", "This App cannot be read right now.") from None
|
| 637 |
if (not isinstance(app, dict) or app.get("id") != app_id or app.get("tenant") != session.tenant
|
| 638 |
+
or app.get("retired") is True or not may_read(app, session.uname, session.admin)):
|
|
|
|
|
|
|
|
|
|
| 639 |
raise err(404, "app_not_found", "This App is unavailable or is not shared with you.")
|
| 640 |
kind = app.get("templateKind")
|
| 641 |
catalogue = _app_templates.BY_KIND.get(str(kind or ""))
|
|
|
|
| 1305 |
|
| 1306 |
|
| 1307 |
def _public(app, user, is_admin=False, *, allowed_databases=None, target_database=None,
|
| 1308 |
+
is_default=False):
|
| 1309 |
row=copy.deepcopy(_app_projection(app))
|
| 1310 |
row.setdefault("origin", None)
|
| 1311 |
if row.get("type") == "code":
|
|
|
|
| 1314 |
compatible = [database for database in row.get("compatibleDatabases") or ()
|
| 1315 |
if database in _allowed(row.get("compatibleDatabases"), allowed_databases)]
|
| 1316 |
row["compatibleDatabases"] = compatible
|
| 1317 |
+
row["capabilities"]={"read":may_read(app,user,is_admin),"edit":may_edit(app,user,is_admin),
|
|
|
|
| 1318 |
"share":may_edit(app,user,is_admin),"delete":may_edit(app,user,is_admin),
|
| 1319 |
+
"duplicate":may_read(app,user,is_admin),"star":may_read(app,user,is_admin),
|
| 1320 |
"default":bool(is_admin)}
|
| 1321 |
row["starred"] = user in (app.get("starredBy") or ())
|
| 1322 |
row["isDefault"] = bool(is_default)
|
|
|
|
| 1377 |
"""List Applications. Unfiltered is the App home; a database filter is how a View resolves."""
|
| 1378 |
doc = load(runtime)
|
| 1379 |
rows=[]
|
|
|
|
| 1380 |
for app in doc["applications"].values():
|
| 1381 |
if (isinstance(app, dict) and getattr(runtime, "key", None)
|
| 1382 |
and app.get("tenant") != runtime.key):
|
|
|
|
| 1386 |
if (isinstance(app, dict) and app.get("templateKind") == "alerts"
|
| 1387 |
and not (may_open or _user_wall(runtime, user))("automation")):
|
| 1388 |
continue
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1389 |
if isinstance(app, dict) and app.get("templateKind") in ("customer_card", FINANCE_TEMPLATE_KIND):
|
| 1390 |
continue
|
| 1391 |
if database:
|
| 1392 |
# A View resolves its App only through this read, so a retired seed stays listed here.
|
| 1393 |
+
if not may_read(app,user,is_admin): continue
|
| 1394 |
if database not in (app.get("compatibleDatabases") or ()): continue
|
| 1395 |
+
elif not _listed(app, user): continue
|
| 1396 |
if (not _database_free(app)
|
| 1397 |
and not _allowed(app.get("compatibleDatabases"), allowed_databases)):
|
| 1398 |
catalogue = _app_templates.BY_KIND.get(str(app.get("templateKind") or ""))
|
|
|
|
| 1404 |
default_key = f"{database}::{app.get('type')}" if database else ""
|
| 1405 |
rows.append(_public(app,user,is_admin,allowed_databases=allowed_databases,
|
| 1406 |
target_database=database,
|
|
|
|
| 1407 |
is_default=bool(default_key and doc["defaults"].get(default_key)==app.get("id"))))
|
| 1408 |
# S67-T39: the dependent-View count rides a DATABASE-filtered read only, and only for rows
|
| 1409 |
# the caller may delete, so the unfiltered nav read (App home tiles) pays nothing for it.
|
api/connectors_amazon.py
CHANGED
|
@@ -611,11 +611,16 @@ def sync_state(rt, tenant_key, *, cadence_seconds=None, now=None):
|
|
| 611 |
answer[table]["warning"] = warning
|
| 612 |
if table == az.SQP_TABLE or table in az.GROUP_A_TABLES or table in ("transactions", "brand_analytics"):
|
| 613 |
answer[table]["limit"] = limit
|
| 614 |
-
|
| 615 |
-
|
| 616 |
-
|
| 617 |
-
|
| 618 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 619 |
# A history table is written in its source's transaction, so it shares the source's last success.
|
| 620 |
for table in az.HISTORY_TABLES:
|
| 621 |
rows = mirrored.get(table)
|
|
|
|
| 611 |
answer[table]["warning"] = warning
|
| 612 |
if table == az.SQP_TABLE or table in az.GROUP_A_TABLES or table in ("transactions", "brand_analytics"):
|
| 613 |
answer[table]["limit"] = limit
|
| 614 |
+
answer["sku_keywords"] = {
|
| 615 |
+
"rowsLoaded": mirrored.get("sku_keywords"),
|
| 616 |
+
"lastSuccess": None,
|
| 617 |
+
"inProgress": False,
|
| 618 |
+
"unknownCause": az.amazon_schema.PROVENANCE_COPY["derived"],
|
| 619 |
+
}
|
| 620 |
+
answer["keyword_ranks"] = {
|
| 621 |
+
"rowsLoaded": mirrored.get("keyword_ranks"), "lastSuccess": None, "inProgress": False,
|
| 622 |
+
"unknownCause": az.amazon_schema.PROVENANCE_COPY["apify_dataset"],
|
| 623 |
+
}
|
| 624 |
# A history table is written in its source's transaction, so it shares the source's last success.
|
| 625 |
for table in az.HISTORY_TABLES:
|
| 626 |
rows = mirrored.get(table)
|
api/external_freshness.py
CHANGED
|
@@ -82,24 +82,18 @@ def _amazon_raw(table_key, source=None):
|
|
| 82 |
return None
|
| 83 |
|
| 84 |
|
| 85 |
-
def amazon_sync_state(cur, rt=None
|
| 86 |
"""`{entity: updated_at}` for every `done` Amazon `_sync_state` row, in one read."""
|
| 87 |
-
if rt is not None
|
| 88 |
from core import connector_sync, connector_spec
|
| 89 |
-
|
| 90 |
-
entry = connector_sync.status(rt, "amazon_sp", fresh=True)
|
| 91 |
if entry is not None:
|
| 92 |
last_pass = entry.get("pass") or {}
|
| 93 |
held = last_pass.get("held") or {}
|
| 94 |
stamps = last_pass.get("tableSync") or {}
|
| 95 |
tables = {row.source.removeprefix("amzn_") for row in connector_spec.AMAZON_TABLES}
|
| 96 |
-
|
| 97 |
-
|
| 98 |
-
if cur is not None and any(name not in stamps for name in held):
|
| 99 |
-
previous = amazon_sync_state(cur)
|
| 100 |
-
state.update({"amzn_sync_mark_" + name: stamp for name in held if name not in stamps
|
| 101 |
-
and (stamp := _amazon_table_stamp(previous, name)) is not None})
|
| 102 |
-
return state
|
| 103 |
return {str(entity): at for entity, at in cur.execute(
|
| 104 |
"SELECT entity, updated_at FROM _sync_state "
|
| 105 |
"WHERE phase = 'done' AND starts_with(entity, 'amzn_')").fetchall()}
|
|
@@ -141,8 +135,8 @@ def amazon_sync_stamp(cur, table_key, state=None):
|
|
| 141 |
return _amazon_table_stamp(state, table)
|
| 142 |
|
| 143 |
|
| 144 |
-
def _amazon_refreshes(session, keys
|
| 145 |
-
"""
|
| 146 |
import assistant_mirror
|
| 147 |
|
| 148 |
cur, state, refreshed = None, None, {}
|
|
@@ -154,7 +148,7 @@ def _amazon_refreshes(session, keys, *, entry, served=False):
|
|
| 154 |
cur = None
|
| 155 |
continue
|
| 156 |
if state is None:
|
| 157 |
-
state = amazon_sync_state(cur,
|
| 158 |
refreshed[key] = amazon_sync_stamp(cur, key, state=state)
|
| 159 |
except Exception:
|
| 160 |
continue
|
|
@@ -244,7 +238,7 @@ def _mirror_refreshes(session, definitions, keys):
|
|
| 244 |
return refreshed
|
| 245 |
|
| 246 |
|
| 247 |
-
def _served_behind(session, key
|
| 248 |
"""process_role's install lag for the connector behind one mirror database, or None."""
|
| 249 |
key = str(key or "")
|
| 250 |
connector = ("amazon" if key.startswith("ut_amzn_") else
|
|
@@ -253,82 +247,49 @@ def _served_behind(session, key, *, worker=None):
|
|
| 253 |
return None
|
| 254 |
try:
|
| 255 |
import process_role
|
| 256 |
-
|
| 257 |
-
worker = process_role.stored_worker(session.runtime)
|
| 258 |
-
return process_role.served_behind(getattr(session.runtime, "key", None) or session.tenant, connector,
|
| 259 |
-
worker=worker)
|
| 260 |
except Exception:
|
| 261 |
return None
|
| 262 |
|
| 263 |
|
| 264 |
-
def
|
| 265 |
-
"""Keep the successful source receipt while naming why its rows have not advanced."""
|
| 266 |
-
fact = {"source_synced_at": utc_stamp(served), "source_sync": state, "source_sync_reason": reason}
|
| 267 |
-
if (day := _sync_date(served)) is not None:
|
| 268 |
-
fact["source_synced_on"] = day
|
| 269 |
-
return fact
|
| 270 |
-
|
| 271 |
-
|
| 272 |
-
def _stale_fact(served, lag=None):
|
| 273 |
"""A fact for a mirror whose newest published sync this deployment could not install."""
|
| 274 |
day = _sync_date(served) or (utc_stamp(served) or "")[:10] or None
|
| 275 |
reason = (f"This deployment still serves the data synced on {day}. The newest sync could not be installed."
|
| 276 |
if day else "This deployment serves an older copy of the data. The newest sync could not be installed.")
|
| 277 |
-
|
| 278 |
-
reason += " " + lag["cause"]
|
| 279 |
-
if lag and (retry := utc_stamp(lag.get("nextRetryAt"))):
|
| 280 |
-
reason += f" Next retry at {retry}."
|
| 281 |
-
return _retained_fact(served, "stale", reason)
|
| 282 |
-
|
| 283 |
-
|
| 284 |
-
def _amazon_holds(entry, keys):
|
| 285 |
-
"""Map each authorized table to its own canonical hold or its held input tables."""
|
| 286 |
-
held = (entry.get("pass") or {}).get("held") or {}
|
| 287 |
-
result = {}
|
| 288 |
-
for key in keys:
|
| 289 |
-
raw = _amazon_raw(key)
|
| 290 |
-
table = raw.removeprefix("amzn_") if raw else ""
|
| 291 |
-
dependencies = (table, *_AMAZON_DERIVED_INPUTS.get(table, ()))
|
| 292 |
-
rows = [held[name] for name in dependencies if isinstance(held.get(name), dict)]
|
| 293 |
-
if rows:
|
| 294 |
-
result[key] = rows
|
| 295 |
-
return result
|
| 296 |
-
|
| 297 |
-
|
| 298 |
-
def _held_fact(served, entries):
|
| 299 |
-
"""Name the kept rows and each stored hold cause without changing their source date."""
|
| 300 |
-
import dash_norm
|
| 301 |
|
| 302 |
-
day = _sync_date(served) or (utc_stamp(served) or "")[:10]
|
| 303 |
-
reason = f"Held at the rows synced on {day}." if day else "Held at the previously synced rows."
|
| 304 |
-
causes = dict.fromkeys(dash_norm.plain(entry.get("cause") or "The last sync did not finish this table.")
|
| 305 |
-
for entry in entries)
|
| 306 |
-
reason += " " + " ".join(causes)
|
| 307 |
-
retries = sorted({stamp for entry in entries if (stamp := utc_stamp(entry.get("nextTryAt")))})
|
| 308 |
-
if retries:
|
| 309 |
-
reason += f" Next retry at {retries[0]}."
|
| 310 |
-
return _retained_fact(served, "stale", reason)
|
| 311 |
|
|
|
|
|
|
|
|
|
|
| 312 |
|
| 313 |
-
|
| 314 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 315 |
edits = {}
|
| 316 |
if edited:
|
|
|
|
|
|
|
|
|
|
| 317 |
try:
|
| 318 |
from core import history_rows
|
| 319 |
latest = history_rows.backend(rt).latest(sorted(edited))
|
| 320 |
-
except Exception:
|
| 321 |
latest = {}
|
| 322 |
for key, at in latest.items() if isinstance(latest, dict) else ():
|
| 323 |
if key in edited:
|
| 324 |
edits[key] = _latest(edits.get(key), at)
|
| 325 |
-
return edits
|
| 326 |
-
|
| 327 |
-
|
| 328 |
-
def _connector_receipts(rt, connected, definitions):
|
| 329 |
-
"""Read matching jobs without borrowing another connection's successful sync."""
|
| 330 |
-
from core import connector_sync
|
| 331 |
-
|
| 332 |
receipts = {}
|
| 333 |
if connected:
|
| 334 |
try:
|
|
@@ -344,84 +305,27 @@ def _connector_receipts(rt, connected, definitions):
|
|
| 344 |
or declared.get("connectionId") and job.get("connectionId") != declared["connectionId"]):
|
| 345 |
continue
|
| 346 |
receipts[key] = job
|
| 347 |
-
return receipts
|
| 348 |
-
|
| 349 |
-
|
| 350 |
-
class _StatusSnapshot:
|
| 351 |
-
"""Keep the request's fresh canonical status available to readers that request freshness again."""
|
| 352 |
-
|
| 353 |
-
def __init__(self, runtime, statuses):
|
| 354 |
-
self._runtime, self._statuses = runtime, statuses
|
| 355 |
-
|
| 356 |
-
def get(self, name, fresh=False):
|
| 357 |
-
from core import connector_sync
|
| 358 |
-
|
| 359 |
-
if name == connector_sync.STATUS_KEY:
|
| 360 |
-
return self._statuses
|
| 361 |
-
return self._runtime.get(name, fresh=True) if fresh else self._runtime.get(name)
|
| 362 |
-
|
| 363 |
-
def __getattr__(self, name):
|
| 364 |
-
return getattr(self._runtime, name)
|
| 365 |
-
|
| 366 |
-
|
| 367 |
-
def _odoo_freshness(rt):
|
| 368 |
-
"""Read Odoo's metadata without restoring or querying its mirror."""
|
| 369 |
-
import odoo_relational
|
| 370 |
-
from core import user_tables
|
| 371 |
-
|
| 372 |
-
docs = {}
|
| 373 |
-
for name in (odoo_relational.CONFIG_KEY, user_tables.ODOO_SYNC_STATUS_KEY):
|
| 374 |
-
try:
|
| 375 |
-
docs[name] = rt.get(name) or {}
|
| 376 |
-
except Exception: # A missing status must not turn a discovery read into an error.
|
| 377 |
-
docs[name] = {}
|
| 378 |
-
return odoo_relational.sync_freshness(user_tables.lend(rt, **docs))
|
| 379 |
-
|
| 380 |
-
|
| 381 |
-
def _synced_fact(values):
|
| 382 |
-
"""Use a successful source timestamp or preserve an explicitly date-only receipt."""
|
| 383 |
-
stamp = next((stamp for value in values if (stamp := utc_stamp(value)) is not None), None)
|
| 384 |
-
if stamp is not None:
|
| 385 |
-
return {"source_synced_at": stamp, "source_sync": "synced"}
|
| 386 |
-
day = next((day for value in values if (day := _sync_date(value)) is not None), None)
|
| 387 |
-
if day is not None:
|
| 388 |
-
return {"source_synced_on": day, "source_sync": "synced_date_only",
|
| 389 |
-
"source_sync_reason": "The source reports only a sync date, without a time."}
|
| 390 |
-
return None
|
| 391 |
-
|
| 392 |
-
|
| 393 |
-
def snapshot(session, databases):
|
| 394 |
-
"""Index source receipts and edit times once for this authorized discovery call."""
|
| 395 |
-
from core import connector_sync, user_tables
|
| 396 |
-
|
| 397 |
-
wanted, rt = set(databases), session.runtime
|
| 398 |
-
try:
|
| 399 |
-
definitions = rt.get_projection(user_tables.STORE_KEY, drop=("rows",))
|
| 400 |
-
if not isinstance(definitions, dict):
|
| 401 |
-
raise ValueError("source definitions unavailable")
|
| 402 |
-
except Exception:
|
| 403 |
-
return {key: {"source_synced_at": None, "source_sync": "never_synced",
|
| 404 |
-
"source_sync_reason": "The source's connector status could not be verified."}
|
| 405 |
-
for key in wanted}
|
| 406 |
-
lent = user_tables.lend(rt, **{user_tables.STORE_KEY: definitions})
|
| 407 |
-
connected = {key for key in wanted if key in definitions and user_tables.is_connected(key, st=lent)}
|
| 408 |
-
edited = {key for key in wanted if key in definitions and key not in connected and key != "ut_activity"}
|
| 409 |
-
edits, receipts = _edit_receipts(rt, edited), _connector_receipts(rt, connected, definitions)
|
| 410 |
odoo_keys = {key for key in connected if key.startswith("ut_odoo_")} | (wanted & {"customer_data", "product_data"})
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 411 |
amazon_keys = {key for key in connected if key.startswith("ut_amzn_")}
|
| 412 |
-
|
| 413 |
-
odoo = _odoo_freshness(_StatusSnapshot(rt, statuses)) if odoo_keys else {}
|
| 414 |
-
worker = (statuses.get("sync_worker") or {}).get("default") or {}
|
| 415 |
-
amazon_entry = (statuses.get("amazon_sp") or {}).get("default")
|
| 416 |
-
behind = {group: _served_behind(session, next(iter(keys)), worker=worker)
|
| 417 |
-
for group, keys in (("odoo", odoo_keys), ("amazon", amazon_keys)) if keys}
|
| 418 |
-
amazon = _amazon_refreshes(session, amazon_keys, entry=amazon_entry,
|
| 419 |
-
served=behind.get("amazon") is not None) if amazon_keys else {}
|
| 420 |
-
held = _amazon_holds(amazon_entry or {}, amazon_keys)
|
| 421 |
mirror_keys = {key for key in connected - amazon_keys
|
| 422 |
if not user_tables.materialises(key, st=lent, defn=definitions[key])}
|
| 423 |
mirror = _mirror_refreshes(session, definitions, mirror_keys) if mirror_keys else {}
|
| 424 |
facts = {}
|
|
|
|
|
|
|
|
|
|
| 425 |
odoo_failed = {str(item.get("key")) for item in (odoo.get("failedDatabases") or ())
|
| 426 |
if isinstance(item, dict) and item.get("key")}
|
| 427 |
for key in wanted:
|
|
@@ -432,10 +336,7 @@ def snapshot(session, databases):
|
|
| 432 |
elif key == "ut_activity":
|
| 433 |
fact["source_sync"] = "not_applicable"
|
| 434 |
elif behind.get("amazon" if key in amazon_keys else "odoo" if key in odoo_keys else None) is not None:
|
| 435 |
-
fact = _stale_fact(amazon.get(key) if key in amazon_keys else mirror.get(key)
|
| 436 |
-
behind["amazon" if key in amazon_keys else "odoo"])
|
| 437 |
-
elif key in held:
|
| 438 |
-
fact = _held_fact(amazon.get(key), held[key])
|
| 439 |
else:
|
| 440 |
health = definition.get(user_tables.CONNECTOR_HEALTH_FIELD) or {}
|
| 441 |
job = receipts.get(key) or {}
|
|
@@ -447,9 +348,13 @@ def snapshot(session, databases):
|
|
| 447 |
if job.get("state") == "ready":
|
| 448 |
successes.append(job.get("at"))
|
| 449 |
values = [amazon.get(key)] if key.startswith("ut_amzn_") else [mirror.get(key), *successes]
|
| 450 |
-
|
| 451 |
-
if
|
| 452 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 453 |
elif (own_failed or job.get("state") == "failed"
|
| 454 |
or key in odoo_keys and odoo.get("phase") == "error"):
|
| 455 |
fact["source_sync_reason"] = "The connector's latest sync failed before a successful sync was recorded."
|
|
|
|
| 82 |
return None
|
| 83 |
|
| 84 |
|
| 85 |
+
def amazon_sync_state(cur, rt=None):
|
| 86 |
"""`{entity: updated_at}` for every `done` Amazon `_sync_state` row, in one read."""
|
| 87 |
+
if rt is not None:
|
| 88 |
from core import connector_sync, connector_spec
|
| 89 |
+
entry = connector_sync.status(rt, "amazon_sp", fresh=True)
|
|
|
|
| 90 |
if entry is not None:
|
| 91 |
last_pass = entry.get("pass") or {}
|
| 92 |
held = last_pass.get("held") or {}
|
| 93 |
stamps = last_pass.get("tableSync") or {}
|
| 94 |
tables = {row.source.removeprefix("amzn_") for row in connector_spec.AMAZON_TABLES}
|
| 95 |
+
return {"amzn_sync_mark_" + name: stamp for name in tables
|
| 96 |
+
if (stamp := stamps.get(name) or (entry.get("lastGoodSyncAt") if name not in held else None))}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 97 |
return {str(entity): at for entity, at in cur.execute(
|
| 98 |
"SELECT entity, updated_at FROM _sync_state "
|
| 99 |
"WHERE phase = 'done' AND starts_with(entity, 'amzn_')").fetchall()}
|
|
|
|
| 135 |
return _amazon_table_stamp(state, table)
|
| 136 |
|
| 137 |
|
| 138 |
+
def _amazon_refreshes(session, keys):
|
| 139 |
+
"""Read only sync metadata for the authorized Amazon databases on this tenant's cursor."""
|
| 140 |
import assistant_mirror
|
| 141 |
|
| 142 |
cur, state, refreshed = None, None, {}
|
|
|
|
| 148 |
cur = None
|
| 149 |
continue
|
| 150 |
if state is None:
|
| 151 |
+
state = amazon_sync_state(cur, session.runtime)
|
| 152 |
refreshed[key] = amazon_sync_stamp(cur, key, state=state)
|
| 153 |
except Exception:
|
| 154 |
continue
|
|
|
|
| 238 |
return refreshed
|
| 239 |
|
| 240 |
|
| 241 |
+
def _served_behind(session, key):
|
| 242 |
"""process_role's install lag for the connector behind one mirror database, or None."""
|
| 243 |
key = str(key or "")
|
| 244 |
connector = ("amazon" if key.startswith("ut_amzn_") else
|
|
|
|
| 247 |
return None
|
| 248 |
try:
|
| 249 |
import process_role
|
| 250 |
+
return process_role.served_behind(getattr(session.runtime, "key", None) or session.tenant, connector)
|
|
|
|
|
|
|
|
|
|
| 251 |
except Exception:
|
| 252 |
return None
|
| 253 |
|
| 254 |
|
| 255 |
+
def _stale_fact(served):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 256 |
"""A fact for a mirror whose newest published sync this deployment could not install."""
|
| 257 |
day = _sync_date(served) or (utc_stamp(served) or "")[:10] or None
|
| 258 |
reason = (f"This deployment still serves the data synced on {day}. The newest sync could not be installed."
|
| 259 |
if day else "This deployment serves an older copy of the data. The newest sync could not be installed.")
|
| 260 |
+
return {"source_synced_at": None, "source_sync": "stale", "source_sync_reason": reason}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 261 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 262 |
|
| 263 |
+
def snapshot(session, databases):
|
| 264 |
+
"""Index source receipts and edit times once for this authorized discovery call."""
|
| 265 |
+
from core import connector_sync, user_tables
|
| 266 |
|
| 267 |
+
wanted = set(databases)
|
| 268 |
+
rt = session.runtime
|
| 269 |
+
try:
|
| 270 |
+
definitions = rt.get_projection(user_tables.STORE_KEY, drop=("rows",))
|
| 271 |
+
if not isinstance(definitions, dict):
|
| 272 |
+
raise ValueError("source definitions unavailable")
|
| 273 |
+
except Exception:
|
| 274 |
+
return {key: {"source_synced_at": None, "source_sync": "never_synced",
|
| 275 |
+
"source_sync_reason": "The source's connector status could not be verified."}
|
| 276 |
+
for key in wanted}
|
| 277 |
+
lent = user_tables.lend(rt, **{user_tables.STORE_KEY: definitions})
|
| 278 |
+
connected = {key for key in wanted if key in definitions and user_tables.is_connected(key, st=lent)}
|
| 279 |
+
edited = {key for key in wanted if key in definitions and key not in connected and key != "ut_activity"}
|
| 280 |
edits = {}
|
| 281 |
if edited:
|
| 282 |
+
# D-11777: edits append only to the history rows, so one SQL aggregate gives each table's
|
| 283 |
+
# newest edit. A store with no history backend (TypeError) or an unreachable one keeps the
|
| 284 |
+
# definition's `updated` stamp below and never fails discovery.
|
| 285 |
try:
|
| 286 |
from core import history_rows
|
| 287 |
latest = history_rows.backend(rt).latest(sorted(edited))
|
| 288 |
+
except Exception:
|
| 289 |
latest = {}
|
| 290 |
for key, at in latest.items() if isinstance(latest, dict) else ():
|
| 291 |
if key in edited:
|
| 292 |
edits[key] = _latest(edits.get(key), at)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 293 |
receipts = {}
|
| 294 |
if connected:
|
| 295 |
try:
|
|
|
|
| 305 |
or declared.get("connectionId") and job.get("connectionId") != declared["connectionId"]):
|
| 306 |
continue
|
| 307 |
receipts[key] = job
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 308 |
odoo_keys = {key for key in connected if key.startswith("ut_odoo_")} | (wanted & {"customer_data", "product_data"})
|
| 309 |
+
odoo = {}
|
| 310 |
+
if odoo_keys:
|
| 311 |
+
import odoo_relational
|
| 312 |
+
|
| 313 |
+
docs = {}
|
| 314 |
+
for name in (odoo_relational.CONFIG_KEY, user_tables.ODOO_SYNC_STATUS_KEY):
|
| 315 |
+
try:
|
| 316 |
+
docs[name] = rt.get(name) or {}
|
| 317 |
+
except Exception:
|
| 318 |
+
docs[name] = {}
|
| 319 |
+
odoo = odoo_relational.sync_freshness(user_tables.lend(rt, **docs))
|
| 320 |
amazon_keys = {key for key in connected if key.startswith("ut_amzn_")}
|
| 321 |
+
amazon = _amazon_refreshes(session, amazon_keys) if amazon_keys else {}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 322 |
mirror_keys = {key for key in connected - amazon_keys
|
| 323 |
if not user_tables.materialises(key, st=lent, defn=definitions[key])}
|
| 324 |
mirror = _mirror_refreshes(session, definitions, mirror_keys) if mirror_keys else {}
|
| 325 |
facts = {}
|
| 326 |
+
# One install-lag read per connector: a secondary web worker reads it from the primary's file.
|
| 327 |
+
behind = {group: _served_behind(session, next(iter(keys)))
|
| 328 |
+
for group, keys in (("odoo", odoo_keys), ("amazon", amazon_keys)) if keys}
|
| 329 |
odoo_failed = {str(item.get("key")) for item in (odoo.get("failedDatabases") or ())
|
| 330 |
if isinstance(item, dict) and item.get("key")}
|
| 331 |
for key in wanted:
|
|
|
|
| 336 |
elif key == "ut_activity":
|
| 337 |
fact["source_sync"] = "not_applicable"
|
| 338 |
elif behind.get("amazon" if key in amazon_keys else "odoo" if key in odoo_keys else None) is not None:
|
| 339 |
+
fact = _stale_fact(amazon.get(key) if key in amazon_keys else mirror.get(key))
|
|
|
|
|
|
|
|
|
|
| 340 |
else:
|
| 341 |
health = definition.get(user_tables.CONNECTOR_HEALTH_FIELD) or {}
|
| 342 |
job = receipts.get(key) or {}
|
|
|
|
| 348 |
if job.get("state") == "ready":
|
| 349 |
successes.append(job.get("at"))
|
| 350 |
values = [amazon.get(key)] if key.startswith("ut_amzn_") else [mirror.get(key), *successes]
|
| 351 |
+
stamp = next((stamp for value in values if (stamp := utc_stamp(value)) is not None), None)
|
| 352 |
+
synced_on = next((day for value in values if (day := _sync_date(value)) is not None), None)
|
| 353 |
+
if stamp is not None:
|
| 354 |
+
fact.update(source_synced_at=stamp, source_sync="synced")
|
| 355 |
+
elif synced_on is not None:
|
| 356 |
+
fact.update(source_synced_on=synced_on, source_sync="synced_date_only",
|
| 357 |
+
source_sync_reason="The source reports only a sync date, without a time.")
|
| 358 |
elif (own_failed or job.get("state") == "failed"
|
| 359 |
or key in odoo_keys and odoo.get("phase") == "error"):
|
| 360 |
fact["source_sync_reason"] = "The connector's latest sync failed before a successful sync was recorded."
|
api/main.py
CHANGED
|
@@ -758,13 +758,6 @@ import app_routes
|
|
| 758 |
_mount_router("routes_security") # W59-T43: lane F declares the router; keep before static mount.
|
| 759 |
_mount_router("routes_amazon_profit") # S78-T03: the Command Center refresh and settings doors.
|
| 760 |
_mount_router("routes_amazon_projects")
|
| 761 |
-
_mount_router("routes_amazon_pdp")
|
| 762 |
-
_mount_router("routes_pdp_text")
|
| 763 |
-
_mount_router("routes_pdp_images")
|
| 764 |
-
if _mount_router("routes_pdp_publish"):
|
| 765 |
-
# Amazon fetches published image URLs without a session, so the media router carries no auth.
|
| 766 |
-
from routes_pdp_publish import media_router as _pdp_media_router
|
| 767 |
-
app.include_router(_pdp_media_router)
|
| 768 |
_mount_router("routes_competitors")
|
| 769 |
_mount_router("routes_odoo_attachments") # S78-T14 (lane G, C5): delivery photo doors; before static mount.
|
| 770 |
_mount_router("routes_sku_actions")
|
|
@@ -1350,8 +1343,6 @@ def _pull_amazon(why, vendor=True):
|
|
| 1350 |
# D-13614: before the lease check, since the chain that holds the lease is the writer that publishes.
|
| 1351 |
_install_amazon_publication(tenant_key, rt, why)
|
| 1352 |
if _amazon.first_load_job_live(rt):
|
| 1353 |
-
from harness import mirror_store
|
| 1354 |
-
mirror_store.record_sync_attempt("amazon", rt)
|
| 1355 |
print("[aios-api] the first-load job holds the Amazon writer lease", flush=True)
|
| 1356 |
return False
|
| 1357 |
# Hotfix 2026-09-24: an unfinished first load goes through the Sync now desk, whose
|
|
@@ -1387,7 +1378,6 @@ def _pull_amazon(why, vendor=True):
|
|
| 1387 |
except Exception as exc:
|
| 1388 |
# A writer that took the lease after the first-load check owns Amazon: skip, record no pass.
|
| 1389 |
if isinstance(exc, RuntimeError) and "holds the amazon writer lease" in str(exc):
|
| 1390 |
-
mirror_store.record_sync_attempt("amazon", rt)
|
| 1391 |
print(f"[aios-api] amazon sync skipped ({why}): another job holds the Amazon writer lease",
|
| 1392 |
flush=True)
|
| 1393 |
return False
|
|
@@ -1759,16 +1749,8 @@ def _amazon_sync_receipt(report, rt=None):
|
|
| 1759 |
|
| 1760 |
|
| 1761 |
def _sync_now_detail(provider, tenant, *, ok, detail="", cause=None):
|
| 1762 |
-
"""
|
| 1763 |
if ok:
|
| 1764 |
-
if provider == "odoo":
|
| 1765 |
-
from harness import mirror_store
|
| 1766 |
-
held = mirror_store.odoo_moving_entities(cause)
|
| 1767 |
-
if held:
|
| 1768 |
-
entity = sorted(held)[0]
|
| 1769 |
-
others = f" {len(held) - 1} other entities were held." if len(held) > 1 else ""
|
| 1770 |
-
return (f"Odoo {entity} changed during all 3 full reads. Its previous rows were kept."
|
| 1771 |
-
+ others + " Other completed entities published.")
|
| 1772 |
return detail
|
| 1773 |
classifier, sentence = _sync_now_cause(cause)
|
| 1774 |
if classifier == "nothing_due":
|
|
@@ -1969,35 +1951,9 @@ def _run_sync_requests(outcome=None, *, during_odoo=False, hold_first_load=None,
|
|
| 1969 |
**({"after_entity": _after_entity} if _after_entity else {}))
|
| 1970 |
if not report and selected_entities != []:
|
| 1971 |
raise _SyncNowRefused("Odoo did not run a sync. Check whether the connection is paused.")
|
| 1972 |
-
|
| 1973 |
-
if any(row.get("phase") != "live" and name not in held for name, row in report.items()):
|
| 1974 |
raise _SyncNowRefused("Odoo backfill is incomplete. Run Sync now again to continue it.")
|
| 1975 |
-
|
| 1976 |
-
if held and waiting == []:
|
| 1977 |
-
import json
|
| 1978 |
-
store = mirror_store.for_connector("odoo")
|
| 1979 |
-
with store.lock:
|
| 1980 |
-
published, _ = store._connect(tenant, rt)
|
| 1981 |
-
candidate = None
|
| 1982 |
-
try:
|
| 1983 |
-
path = rt.datastore_path.with_suffix(".odoo.candidate.duckdb")
|
| 1984 |
-
candidate = datastore.apply_limits(datastore.duckdb.connect(str(path), read_only=True))
|
| 1985 |
-
retained = {row[0] for row in candidate.execute("SELECT name FROM _candidate_held").fetchall()}
|
| 1986 |
-
publication = json.loads(store._download(rt, mirror_store._PASS_KEY) or b"{}")
|
| 1987 |
-
installed = datastore._publication(published, "odoo")
|
| 1988 |
-
if (retained == set(held) and publication.get("state") == "published"
|
| 1989 |
-
and publication.get("tenant") == tenant and publication.get("connector") == "odoo"
|
| 1990 |
-
and publication.get("completed_at") and installed.get("state") == "complete"
|
| 1991 |
-
and publication.get("snapshot_id")
|
| 1992 |
-
and publication.get("snapshot_id") == installed.get("snapshot_id")):
|
| 1993 |
-
waiting = retained
|
| 1994 |
-
finally:
|
| 1995 |
-
if candidate is not None:
|
| 1996 |
-
candidate.close()
|
| 1997 |
-
published.close()
|
| 1998 |
-
if ((held and (set(waiting or ()) != set(held)
|
| 1999 |
-
or not any(row.get("phase") == "live" for row in report.values())))
|
| 2000 |
-
or (not held and waiting is not None)):
|
| 2001 |
raise _SyncNowRefused("Odoo publication is held. The previous snapshot remains available.")
|
| 2002 |
problems = []
|
| 2003 |
elif provider == "amazon_sp":
|
|
@@ -2090,15 +2046,14 @@ def _run_sync_requests(outcome=None, *, during_odoo=False, hold_first_load=None,
|
|
| 2090 |
detail = _sync_now_detail(provider, tenant, ok=False, detail=amazon_detail,
|
| 2091 |
cause=amazon_failure or problems[0])
|
| 2092 |
else:
|
| 2093 |
-
ok, detail = True,
|
| 2094 |
-
cause=report if provider == "odoo" else None)
|
| 2095 |
if amazon_detail and not problems:
|
| 2096 |
detail += " " + amazon_detail
|
| 2097 |
except _SyncNowRefused as exc:
|
| 2098 |
detail = str(exc)
|
| 2099 |
except Exception as exc: # noqa: BLE001
|
| 2100 |
detail = _sync_now_detail(provider, tenant, ok=False, cause=exc)
|
| 2101 |
-
if progress and not amazon_detail
|
| 2102 |
import re
|
| 2103 |
suffix = re.search(r" \(cause code: [a-z_]+\)$", detail)
|
| 2104 |
cause_code = suffix[0] if suffix else ""
|
|
@@ -3565,8 +3520,6 @@ def _amazon_desk_pass(tenant, *, project=True):
|
|
| 3565 |
job.join()
|
| 3566 |
state["job"] = None
|
| 3567 |
_project_desk(state["outcome"], tenant)
|
| 3568 |
-
import sync_runtime
|
| 3569 |
-
sync_runtime.relay_at_boundary()
|
| 3570 |
receipt = connector_sync.sync_request("amazon_sp", tenant)
|
| 3571 |
if (not receipt or receipt["state"] != "queued" or receipt.get("requestedBy") == "first load"
|
| 3572 |
or max(receipt.get("notBefore") or 0,
|
|
@@ -3884,6 +3837,7 @@ def _store_resync_loop():
|
|
| 3884 |
_resync_status = globals().get("_publish_resync_state")
|
| 3885 |
if _resync_status:
|
| 3886 |
_resync_status(_cad["tenant"], started=True)
|
|
|
|
| 3887 |
try:
|
| 3888 |
selected_entities = _configured_mirror_entities(_cad["tenant"])
|
| 3889 |
sync_report, delete_report, _err = _resync_mirror_pass(_manual, passes, selected_entities)
|
|
@@ -3914,6 +3868,26 @@ def _store_resync_loop():
|
|
| 3914 |
"source", "ready", at=RESYNC_STATE["lastPassAt"], cadence_seconds=_every)
|
| 3915 |
except Exception: # noqa: BLE001
|
| 3916 |
pass # The resident mirror state above still reports this pass.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3917 |
else:
|
| 3918 |
RESYNC_STATE["consecutiveFailures"] = int(RESYNC_STATE["consecutiveFailures"]) + 1
|
| 3919 |
# D-314: the TYPE and the persistent/transient reading, both in the line and in the
|
|
|
|
| 758 |
_mount_router("routes_security") # W59-T43: lane F declares the router; keep before static mount.
|
| 759 |
_mount_router("routes_amazon_profit") # S78-T03: the Command Center refresh and settings doors.
|
| 760 |
_mount_router("routes_amazon_projects")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 761 |
_mount_router("routes_competitors")
|
| 762 |
_mount_router("routes_odoo_attachments") # S78-T14 (lane G, C5): delivery photo doors; before static mount.
|
| 763 |
_mount_router("routes_sku_actions")
|
|
|
|
| 1343 |
# D-13614: before the lease check, since the chain that holds the lease is the writer that publishes.
|
| 1344 |
_install_amazon_publication(tenant_key, rt, why)
|
| 1345 |
if _amazon.first_load_job_live(rt):
|
|
|
|
|
|
|
| 1346 |
print("[aios-api] the first-load job holds the Amazon writer lease", flush=True)
|
| 1347 |
return False
|
| 1348 |
# Hotfix 2026-09-24: an unfinished first load goes through the Sync now desk, whose
|
|
|
|
| 1378 |
except Exception as exc:
|
| 1379 |
# A writer that took the lease after the first-load check owns Amazon: skip, record no pass.
|
| 1380 |
if isinstance(exc, RuntimeError) and "holds the amazon writer lease" in str(exc):
|
|
|
|
| 1381 |
print(f"[aios-api] amazon sync skipped ({why}): another job holds the Amazon writer lease",
|
| 1382 |
flush=True)
|
| 1383 |
return False
|
|
|
|
| 1749 |
|
| 1750 |
|
| 1751 |
def _sync_now_detail(provider, tenant, *, ok, detail="", cause=None):
|
| 1752 |
+
"""Record one classified failure; leave successful receipt text untouched."""
|
| 1753 |
if ok:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1754 |
return detail
|
| 1755 |
classifier, sentence = _sync_now_cause(cause)
|
| 1756 |
if classifier == "nothing_due":
|
|
|
|
| 1951 |
**({"after_entity": _after_entity} if _after_entity else {}))
|
| 1952 |
if not report and selected_entities != []:
|
| 1953 |
raise _SyncNowRefused("Odoo did not run a sync. Check whether the connection is paused.")
|
| 1954 |
+
if any(row.get("phase") != "live" for row in report.values()):
|
|
|
|
| 1955 |
raise _SyncNowRefused("Odoo backfill is incomplete. Run Sync now again to continue it.")
|
| 1956 |
+
if mirror_store.pending_sync_entities("odoo", tenant, rt) is not None:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1957 |
raise _SyncNowRefused("Odoo publication is held. The previous snapshot remains available.")
|
| 1958 |
problems = []
|
| 1959 |
elif provider == "amazon_sp":
|
|
|
|
| 2046 |
detail = _sync_now_detail(provider, tenant, ok=False, detail=amazon_detail,
|
| 2047 |
cause=amazon_failure or problems[0])
|
| 2048 |
else:
|
| 2049 |
+
ok, detail = True, "Sync finished."
|
|
|
|
| 2050 |
if amazon_detail and not problems:
|
| 2051 |
detail += " " + amazon_detail
|
| 2052 |
except _SyncNowRefused as exc:
|
| 2053 |
detail = str(exc)
|
| 2054 |
except Exception as exc: # noqa: BLE001
|
| 2055 |
detail = _sync_now_detail(provider, tenant, ok=False, cause=exc)
|
| 2056 |
+
if progress and not amazon_detail:
|
| 2057 |
import re
|
| 2058 |
suffix = re.search(r" \(cause code: [a-z_]+\)$", detail)
|
| 2059 |
cause_code = suffix[0] if suffix else ""
|
|
|
|
| 3520 |
job.join()
|
| 3521 |
state["job"] = None
|
| 3522 |
_project_desk(state["outcome"], tenant)
|
|
|
|
|
|
|
| 3523 |
receipt = connector_sync.sync_request("amazon_sp", tenant)
|
| 3524 |
if (not receipt or receipt["state"] != "queued" or receipt.get("requestedBy") == "first load"
|
| 3525 |
or max(receipt.get("notBefore") or 0,
|
|
|
|
| 3837 |
_resync_status = globals().get("_publish_resync_state")
|
| 3838 |
if _resync_status:
|
| 3839 |
_resync_status(_cad["tenant"], started=True)
|
| 3840 |
+
_pass_started_epoch = _t.time()
|
| 3841 |
try:
|
| 3842 |
selected_entities = _configured_mirror_entities(_cad["tenant"])
|
| 3843 |
sync_report, delete_report, _err = _resync_mirror_pass(_manual, passes, selected_entities)
|
|
|
|
| 3868 |
"source", "ready", at=RESYNC_STATE["lastPassAt"], cadence_seconds=_every)
|
| 3869 |
except Exception: # noqa: BLE001
|
| 3870 |
pass # The resident mirror state above still reports this pass.
|
| 3871 |
+
elif not _manual:
|
| 3872 |
+
# A held pass that published this pass's snapshot is a success. A refused publication writes nothing.
|
| 3873 |
+
try:
|
| 3874 |
+
from datetime import datetime as _held_dt
|
| 3875 |
+
from harness import runtime as _held_runtime
|
| 3876 |
+
import routes_connectors as _held_receipts
|
| 3877 |
+
_held_rt = _held_runtime.get_runtime(_cad["tenant"])
|
| 3878 |
+
_held_pass = (connector_sync.status(
|
| 3879 |
+
_held_rt, "odoo", RESYNC_STATE.get("connectionId") or "default",
|
| 3880 |
+
fresh=True) or {}).get("pass") or {}
|
| 3881 |
+
_published = (connector_sync._utc_stamp(_held_pass["publishedAt"])
|
| 3882 |
+
if _held_pass.get("publishedAt") else None)
|
| 3883 |
+
if _published and (_held_dt.fromisoformat(_published).timestamp()
|
| 3884 |
+
>= int(_pass_started_epoch)):
|
| 3885 |
+
RESYNC_STATE["lastSuccessAt"] = RESYNC_STATE["lastPassAt"]
|
| 3886 |
+
_held_receipts.record_odoo_stage(
|
| 3887 |
+
_held_rt, RESYNC_STATE.get("connectionId") or "default",
|
| 3888 |
+
"source", "ready", at=_published, cadence_seconds=_every)
|
| 3889 |
+
except Exception: # noqa: BLE001
|
| 3890 |
+
pass # The pass row in the status document still reports this publication.
|
| 3891 |
else:
|
| 3892 |
RESYNC_STATE["consecutiveFailures"] = int(RESYNC_STATE["consecutiveFailures"]) + 1
|
| 3893 |
# D-314: the TYPE and the persistent/transient reading, both in the line and in the
|
api/odoo_relational.py
CHANGED
|
@@ -6483,7 +6483,13 @@ def sync_freshness(rt, now=None):
|
|
| 6483 |
status = status if isinstance(status, dict) else {}
|
| 6484 |
from core import connector_sync
|
| 6485 |
reader = getattr(connector_sync, "status", None)
|
| 6486 |
-
canonical =
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6487 |
canonical = canonical if isinstance(canonical, dict) else None
|
| 6488 |
pass_status = _sync_pass_freshness(canonical, cfg, now)
|
| 6489 |
phase = str(status.get("phase") or "unknown")
|
|
|
|
| 6483 |
status = status if isinstance(status, dict) else {}
|
| 6484 |
from core import connector_sync
|
| 6485 |
reader = getattr(connector_sync, "status", None)
|
| 6486 |
+
canonical = None
|
| 6487 |
+
if reader is not None and rt is not None:
|
| 6488 |
+
# The resync loop writes under the keychain default id. Read that slot first, else "default".
|
| 6489 |
+
default_id = default_connection_id(rt)
|
| 6490 |
+
canonical = reader(rt, "odoo", default_id, fresh=True) if default_id else None
|
| 6491 |
+
if not isinstance(canonical, dict):
|
| 6492 |
+
canonical = reader(rt, "odoo", fresh=True)
|
| 6493 |
canonical = canonical if isinstance(canonical, dict) else None
|
| 6494 |
pass_status = _sync_pass_freshness(canonical, cfg, now)
|
| 6495 |
phase = str(status.get("phase") or "unknown")
|
api/process_role.py
CHANGED
|
@@ -56,20 +56,6 @@ READY_FILE = "READY.json"
|
|
| 56 |
PRIMARY_LOCK = "web-primary.lock"
|
| 57 |
PRIMARY_FILE = "web-primary.json"
|
| 58 |
INSTALL_ATTEMPTS = 3
|
| 59 |
-
INSTALL_RETRY_SECONDS = 3600
|
| 60 |
-
_INSTALL_CAUSES = (
|
| 61 |
-
(TimeoutError, "The install timed out."),
|
| 62 |
-
(ConnectionError, "The saved sync could not be downloaded."),
|
| 63 |
-
(PermissionError, "The install could not access the saved sync."),
|
| 64 |
-
(FileNotFoundError, "The saved sync is missing a required file."),
|
| 65 |
-
(OSError, "The install could not read or write its files."),
|
| 66 |
-
(MemoryError, "The install ran out of memory."),
|
| 67 |
-
(ValueError, "The saved sync failed validation."),
|
| 68 |
-
(TypeError, "The saved sync has an invalid structure."),
|
| 69 |
-
(LookupError, "The saved sync is missing required data."),
|
| 70 |
-
(RuntimeError, "The install stopped with an internal error."),
|
| 71 |
-
(Exception, "The install failed unexpectedly."),
|
| 72 |
-
)
|
| 73 |
INCOMPLETE_GRACE_SECONDS = 3600
|
| 74 |
#: The C-status provider key the sync worker writes its heartbeat, catch-up and mirror ids under.
|
| 75 |
WORKER_STATUS = "sync_worker"
|
|
@@ -550,34 +536,37 @@ def pending_handoffs(root=None):
|
|
| 550 |
|
| 551 |
|
| 552 |
def _failure_map(root=None):
|
| 553 |
-
"""
|
| 554 |
if not is_primary():
|
| 555 |
stored = primary_state(root).get("installFailures") or {}
|
| 556 |
return {tuple(key.split("/", 1)): row for key, row in stored.items()
|
| 557 |
if "/" in key and isinstance(row, dict)}
|
| 558 |
-
|
| 559 |
-
|
| 560 |
-
|
| 561 |
-
|
| 562 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 563 |
"""`{published, served, attempts, error}` when the store published a snapshot this container failed to install.
|
| 564 |
|
| 565 |
None when nothing is published, the published snapshot is served, or no install of it failed yet.
|
| 566 |
"""
|
| 567 |
key = (str(tenant or "").strip().lower(), str(connector))
|
| 568 |
-
|
| 569 |
-
published = ((worker.get("mirrors") or {}).get(key[1]) or {}).get("snapshotId") or _known_mirrors(root).get(key)
|
| 570 |
if not published:
|
| 571 |
return None
|
| 572 |
served = _installed_map(root).get(key)
|
| 573 |
-
failure = _failure_map(root).get(key) or
|
| 574 |
-
if (failure.get("tenant"), failure.get("connector"), failure.get("snapshotId")) != (*key, published):
|
| 575 |
-
return None
|
| 576 |
attempts = int(failure.get("attempts") or 0) if failure.get("snapshotId") == published else 0
|
| 577 |
if served == published or attempts < 1:
|
| 578 |
return None
|
| 579 |
-
return {"published": published, "served": served
|
| 580 |
-
"error": failure.get("type"), "cause": failure.get("cause"), "nextRetryAt": failure.get("nextRetryAt")}
|
| 581 |
|
| 582 |
|
| 583 |
# ----------------------------------------------------------------------------- several web workers
|
|
@@ -785,44 +774,6 @@ def _remove_dir(path):
|
|
| 785 |
shutil.rmtree(path, ignore_errors=True)
|
| 786 |
|
| 787 |
|
| 788 |
-
def _install_key(key, snapshot):
|
| 789 |
-
return "/".join((*key, snapshot))
|
| 790 |
-
|
| 791 |
-
|
| 792 |
-
def _install_due(key, snapshot, stamp):
|
| 793 |
-
failure = _INSTALL_FAILURES.get((key, snapshot)) or {}
|
| 794 |
-
return failure.get("attempts", 0) < INSTALL_ATTEMPTS or stamp >= failure["nextRetryAt"]
|
| 795 |
-
|
| 796 |
-
|
| 797 |
-
def _record_install(key, snapshot, stamp, *, exc=None, rt=None, changed=True):
|
| 798 |
-
"""Keep one scoped install receipt and persist its safe cause and retry deadline."""
|
| 799 |
-
from core import connector_sync
|
| 800 |
-
|
| 801 |
-
identity = (key, snapshot)
|
| 802 |
-
was_failed = identity in _INSTALL_FAILURES
|
| 803 |
-
row = {"tenant": key[0], "connector": key[1], "snapshotId": snapshot, "at": stamp,
|
| 804 |
-
"state": "failed" if exc is not None else "installed"}
|
| 805 |
-
if exc is not None:
|
| 806 |
-
kind, cause = next((kind, cause) for kind, cause in _INSTALL_CAUSES if isinstance(exc, kind))
|
| 807 |
-
attempts = (_INSTALL_FAILURES.get(identity) or {}).get("attempts", 0) + 1
|
| 808 |
-
row.update(attempts=attempts, type=kind.__name__, cause=cause, served=_INSTALLED.get(key),
|
| 809 |
-
nextRetryAt=stamp + (INSTALL_RETRY_SECONDS if attempts >= INSTALL_ATTEMPTS else POLL_SECONDS))
|
| 810 |
-
_INSTALL_FAILURES[identity] = row
|
| 811 |
-
LAST_INSTALL_ERROR.clear()
|
| 812 |
-
LAST_INSTALL_ERROR.update(row)
|
| 813 |
-
else:
|
| 814 |
-
row["changed"] = changed is not False
|
| 815 |
-
_INSTALL_FAILURES.pop(identity, None)
|
| 816 |
-
if (LAST_INSTALL_ERROR.get("tenant"), LAST_INSTALL_ERROR.get("connector"),
|
| 817 |
-
LAST_INSTALL_ERROR.get("snapshotId")) == (*key, snapshot):
|
| 818 |
-
LAST_INSTALL_ERROR.clear()
|
| 819 |
-
if rt is not None and (exc is not None or was_failed):
|
| 820 |
-
written = connector_sync.update_status(rt, WORKER_STATUS, "installs", {_install_key(key, snapshot): row})
|
| 821 |
-
if not written:
|
| 822 |
-
print(f"[process-role] install status not stored for {key[0]}/{key[1]}", flush=True)
|
| 823 |
-
return row
|
| 824 |
-
|
| 825 |
-
|
| 826 |
def install_ready(root=None, installer=None, *, now=None):
|
| 827 |
"""Install the newest READY snapshot per tenant and connector, then delete older hand-offs.
|
| 828 |
|
|
@@ -835,22 +786,25 @@ def install_ready(root=None, installer=None, *, now=None):
|
|
| 835 |
for key, rows in ready_handoffs(root).items():
|
| 836 |
completed_at, newest_dir, ready = rows[-1]
|
| 837 |
snapshot = ready.get("snapshotId") or newest_dir.name
|
| 838 |
-
if _INSTALLED.get(key) != snapshot and
|
| 839 |
try:
|
| 840 |
changed = installer(key[1], key[0], ready)
|
| 841 |
-
except Exception as exc:
|
| 842 |
-
|
|
|
|
|
|
|
| 843 |
print(f"[process-role] hand-off install {key[0]}/{key[1]} {snapshot} failed "
|
| 844 |
-
f"({
|
| 845 |
continue
|
| 846 |
_INSTALLED[key] = snapshot
|
| 847 |
-
_record_install(key, snapshot, stamp, changed=changed)
|
| 848 |
if changed is False:
|
| 849 |
# Already resident: nothing was rebuilt, so this is not an install.
|
| 850 |
for _at, older, _ready in rows[:-1]:
|
| 851 |
_remove_dir(older)
|
| 852 |
continue
|
| 853 |
LAST_INSTALL.update(tenant=key[0], connector=key[1], snapshotId=snapshot, at=stamp)
|
|
|
|
|
|
|
| 854 |
installed.append(key)
|
| 855 |
print(f"[process-role] installed {key[1]} snapshot {snapshot} for {key[0]}", flush=True)
|
| 856 |
for _at, older, _ready in rows[:-1]:
|
|
@@ -908,31 +862,29 @@ def install_stored(tenant, rt, mirrors, installer=None, *, now=None):
|
|
| 908 |
installer = installer or _install_from_store
|
| 909 |
stamp = time.time() if now is None else now
|
| 910 |
installed = []
|
| 911 |
-
stored = (stored_worker(rt).get("installs") or {}) if rt is not None else {}
|
| 912 |
for connector, row in sorted((mirrors or {}).items()):
|
| 913 |
snapshot = row.get("snapshotId") if isinstance(row, dict) else None
|
| 914 |
if not snapshot:
|
| 915 |
continue
|
| 916 |
-
key = (str(tenant)
|
| 917 |
_STORED["mirrors"][key] = snapshot
|
| 918 |
-
|
| 919 |
-
if (previous.get("state") == "failed" and (key, snapshot) not in _INSTALL_FAILURES
|
| 920 |
-
and (previous.get("tenant"), previous.get("connector"), previous.get("snapshotId")) == (*key, snapshot)):
|
| 921 |
-
_INSTALL_FAILURES[(key, snapshot)] = previous
|
| 922 |
-
if _INSTALLED.get(key) == snapshot or not _install_due(key, snapshot, stamp):
|
| 923 |
continue
|
| 924 |
try:
|
| 925 |
changed = installer(key[1], key[0], rt, snapshot)
|
| 926 |
-
except Exception as exc:
|
| 927 |
-
|
|
|
|
|
|
|
| 928 |
print(f"[process-role] store install {key[0]}/{key[1]} {snapshot} failed "
|
| 929 |
-
f"({
|
| 930 |
continue
|
| 931 |
_INSTALLED[key] = snapshot
|
| 932 |
-
_record_install(key, snapshot, stamp, rt=rt, changed=changed)
|
| 933 |
if changed is False:
|
| 934 |
continue
|
| 935 |
LAST_INSTALL.update(tenant=key[0], connector=key[1], snapshotId=snapshot, at=stamp)
|
|
|
|
|
|
|
| 936 |
installed.append(key)
|
| 937 |
print(f"[process-role] installed {key[1]} snapshot {snapshot} for {key[0]} from the store", flush=True)
|
| 938 |
return installed
|
|
@@ -1283,23 +1235,6 @@ def start_web_services():
|
|
| 1283 |
return True
|
| 1284 |
|
| 1285 |
|
| 1286 |
-
def _install_diagnostics(stored):
|
| 1287 |
-
"""Read this tenant's current failed snapshot and last changed install across web processes."""
|
| 1288 |
-
tenant = writer_tenant()
|
| 1289 |
-
rows = {**(stored.get("installs") or {}), **{
|
| 1290 |
-
_install_key(key, snapshot): row for (key, snapshot), row in dict(_INSTALL_FAILURES).items()}}
|
| 1291 |
-
rows = sorted((row for row in rows.values() if isinstance(row, dict) and row.get("tenant") == tenant),
|
| 1292 |
-
key=lambda row: row.get("at", 0), reverse=True)
|
| 1293 |
-
mirrors = {key[1]: snapshot for key, snapshot in _known_mirrors().items() if key[0] == tenant}
|
| 1294 |
-
mirrors.update({connector: row.get("snapshotId") for connector, row in (stored.get("mirrors") or {}).items()})
|
| 1295 |
-
failure = next((row for row in rows if row.get("state") == "failed"
|
| 1296 |
-
and mirrors.get(row["connector"]) == row["snapshotId"]
|
| 1297 |
-
and _INSTALLED.get((tenant, row["connector"])) != row["snapshotId"]), None)
|
| 1298 |
-
installed = next((row for row in rows if row.get("state") == "installed" and row.get("changed")), None)
|
| 1299 |
-
local = dict(LAST_INSTALL) if LAST_INSTALL.get("tenant") == tenant else None
|
| 1300 |
-
return local or installed, failure
|
| 1301 |
-
|
| 1302 |
-
|
| 1303 |
def status():
|
| 1304 |
"""{role, worker: {pid, alive, heartbeatAgeS, restarts, lastInstall}} for the settings response.
|
| 1305 |
|
|
@@ -1324,13 +1259,12 @@ def status():
|
|
| 1324 |
fleet[slug] = {"pid": None, "alive": bool(row.get("alive")), "restarts": row.get("jobs", 0),
|
| 1325 |
"lastExit": None, "jobId": row.get("jobId")}
|
| 1326 |
import sync_runtime
|
| 1327 |
-
last_install, last_error = _install_diagnostics(stored)
|
| 1328 |
return {"role": ROLE, "roleSource": ROLE_SOURCE, "webWorkers": web_workers(), "primary": is_primary(),
|
| 1329 |
"syncRuntime": sync_runtime.runtime_name()[0],
|
| 1330 |
"tenant": writer_tenant(), "scope": worker_scope(), "tenantWorkers": fleet,
|
| 1331 |
"tenantWorkersOverCap": list(_FLEET_STATE["overCap"]), "syncWorkersMax": sync_worker_cap(),
|
| 1332 |
"worker": {"pid": pid, "alive": alive, "heartbeatAgeS": age,
|
| 1333 |
-
"restarts": _WORKER["restarts"], "lastInstall":
|
| 1334 |
-
"lastInstallError":
|
| 1335 |
"attestation": worker_provenance.visible(stored) if stored.get("heartbeat") == beat else None},
|
| 1336 |
"writer": {"armed": _WRITER["armed"], "held": _WRITER_HELD.is_set(), "holder": _WRITER["holder"]}}
|
|
|
|
| 56 |
PRIMARY_LOCK = "web-primary.lock"
|
| 57 |
PRIMARY_FILE = "web-primary.json"
|
| 58 |
INSTALL_ATTEMPTS = 3
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 59 |
INCOMPLETE_GRACE_SECONDS = 3600
|
| 60 |
#: The C-status provider key the sync worker writes its heartbeat, catch-up and mirror ids under.
|
| 61 |
WORKER_STATUS = "sync_worker"
|
|
|
|
| 536 |
|
| 537 |
|
| 538 |
def _failure_map(root=None):
|
| 539 |
+
"""{(tenant, connector): {snapshotId, attempts, type, at}} for each published snapshot that failed to install."""
|
| 540 |
if not is_primary():
|
| 541 |
stored = primary_state(root).get("installFailures") or {}
|
| 542 |
return {tuple(key.split("/", 1)): row for key, row in stored.items()
|
| 543 |
if "/" in key and isinstance(row, dict)}
|
| 544 |
+
last, failures = dict(LAST_INSTALL_ERROR), {}
|
| 545 |
+
for key, snapshot in dict(_STORED["mirrors"]).items():
|
| 546 |
+
attempts = _INSTALL_FAILURES.get((key, snapshot), 0)
|
| 547 |
+
if attempts:
|
| 548 |
+
# LAST_INSTALL_ERROR is one slot: its type and time belong only to the snapshot it names.
|
| 549 |
+
same = (last.get("tenant"), last.get("connector"), last.get("snapshotId")) == (*key, snapshot)
|
| 550 |
+
failures[key] = {"snapshotId": snapshot, "attempts": attempts,
|
| 551 |
+
"type": last.get("type") if same else None, "at": last.get("at") if same else None}
|
| 552 |
+
return failures
|
| 553 |
+
|
| 554 |
+
|
| 555 |
+
def served_behind(tenant, connector, root=None):
|
| 556 |
"""`{published, served, attempts, error}` when the store published a snapshot this container failed to install.
|
| 557 |
|
| 558 |
None when nothing is published, the published snapshot is served, or no install of it failed yet.
|
| 559 |
"""
|
| 560 |
key = (str(tenant or "").strip().lower(), str(connector))
|
| 561 |
+
published = _known_mirrors(root).get(key)
|
|
|
|
| 562 |
if not published:
|
| 563 |
return None
|
| 564 |
served = _installed_map(root).get(key)
|
| 565 |
+
failure = _failure_map(root).get(key) or {}
|
|
|
|
|
|
|
| 566 |
attempts = int(failure.get("attempts") or 0) if failure.get("snapshotId") == published else 0
|
| 567 |
if served == published or attempts < 1:
|
| 568 |
return None
|
| 569 |
+
return {"published": published, "served": served, "attempts": attempts, "error": failure.get("type")}
|
|
|
|
| 570 |
|
| 571 |
|
| 572 |
# ----------------------------------------------------------------------------- several web workers
|
|
|
|
| 774 |
shutil.rmtree(path, ignore_errors=True)
|
| 775 |
|
| 776 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 777 |
def install_ready(root=None, installer=None, *, now=None):
|
| 778 |
"""Install the newest READY snapshot per tenant and connector, then delete older hand-offs.
|
| 779 |
|
|
|
|
| 786 |
for key, rows in ready_handoffs(root).items():
|
| 787 |
completed_at, newest_dir, ready = rows[-1]
|
| 788 |
snapshot = ready.get("snapshotId") or newest_dir.name
|
| 789 |
+
if _INSTALLED.get(key) != snapshot and _INSTALL_FAILURES.get((key, snapshot), 0) < INSTALL_ATTEMPTS:
|
| 790 |
try:
|
| 791 |
changed = installer(key[1], key[0], ready)
|
| 792 |
+
except Exception as exc: # noqa: BLE001
|
| 793 |
+
_INSTALL_FAILURES[(key, snapshot)] = _INSTALL_FAILURES.get((key, snapshot), 0) + 1
|
| 794 |
+
LAST_INSTALL_ERROR.update(tenant=key[0], connector=key[1], snapshotId=snapshot, at=stamp,
|
| 795 |
+
attempts=_INSTALL_FAILURES[(key, snapshot)], type=type(exc).__name__)
|
| 796 |
print(f"[process-role] hand-off install {key[0]}/{key[1]} {snapshot} failed "
|
| 797 |
+
f"({type(exc).__name__}: {exc})", flush=True)
|
| 798 |
continue
|
| 799 |
_INSTALLED[key] = snapshot
|
|
|
|
| 800 |
if changed is False:
|
| 801 |
# Already resident: nothing was rebuilt, so this is not an install.
|
| 802 |
for _at, older, _ready in rows[:-1]:
|
| 803 |
_remove_dir(older)
|
| 804 |
continue
|
| 805 |
LAST_INSTALL.update(tenant=key[0], connector=key[1], snapshotId=snapshot, at=stamp)
|
| 806 |
+
if (LAST_INSTALL_ERROR.get("tenant"), LAST_INSTALL_ERROR.get("connector")) == key:
|
| 807 |
+
LAST_INSTALL_ERROR.clear()
|
| 808 |
installed.append(key)
|
| 809 |
print(f"[process-role] installed {key[1]} snapshot {snapshot} for {key[0]}", flush=True)
|
| 810 |
for _at, older, _ready in rows[:-1]:
|
|
|
|
| 862 |
installer = installer or _install_from_store
|
| 863 |
stamp = time.time() if now is None else now
|
| 864 |
installed = []
|
|
|
|
| 865 |
for connector, row in sorted((mirrors or {}).items()):
|
| 866 |
snapshot = row.get("snapshotId") if isinstance(row, dict) else None
|
| 867 |
if not snapshot:
|
| 868 |
continue
|
| 869 |
+
key = (str(tenant), str(connector))
|
| 870 |
_STORED["mirrors"][key] = snapshot
|
| 871 |
+
if _INSTALLED.get(key) == snapshot or _INSTALL_FAILURES.get((key, snapshot), 0) >= INSTALL_ATTEMPTS:
|
|
|
|
|
|
|
|
|
|
|
|
|
| 872 |
continue
|
| 873 |
try:
|
| 874 |
changed = installer(key[1], key[0], rt, snapshot)
|
| 875 |
+
except Exception as exc: # noqa: BLE001
|
| 876 |
+
_INSTALL_FAILURES[(key, snapshot)] = _INSTALL_FAILURES.get((key, snapshot), 0) + 1
|
| 877 |
+
LAST_INSTALL_ERROR.update(tenant=key[0], connector=key[1], snapshotId=snapshot, at=stamp,
|
| 878 |
+
attempts=_INSTALL_FAILURES[(key, snapshot)], type=type(exc).__name__)
|
| 879 |
print(f"[process-role] store install {key[0]}/{key[1]} {snapshot} failed "
|
| 880 |
+
f"({type(exc).__name__}: {exc})", flush=True)
|
| 881 |
continue
|
| 882 |
_INSTALLED[key] = snapshot
|
|
|
|
| 883 |
if changed is False:
|
| 884 |
continue
|
| 885 |
LAST_INSTALL.update(tenant=key[0], connector=key[1], snapshotId=snapshot, at=stamp)
|
| 886 |
+
if (LAST_INSTALL_ERROR.get("tenant"), LAST_INSTALL_ERROR.get("connector")) == key:
|
| 887 |
+
LAST_INSTALL_ERROR.clear()
|
| 888 |
installed.append(key)
|
| 889 |
print(f"[process-role] installed {key[1]} snapshot {snapshot} for {key[0]} from the store", flush=True)
|
| 890 |
return installed
|
|
|
|
| 1235 |
return True
|
| 1236 |
|
| 1237 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1238 |
def status():
|
| 1239 |
"""{role, worker: {pid, alive, heartbeatAgeS, restarts, lastInstall}} for the settings response.
|
| 1240 |
|
|
|
|
| 1259 |
fleet[slug] = {"pid": None, "alive": bool(row.get("alive")), "restarts": row.get("jobs", 0),
|
| 1260 |
"lastExit": None, "jobId": row.get("jobId")}
|
| 1261 |
import sync_runtime
|
|
|
|
| 1262 |
return {"role": ROLE, "roleSource": ROLE_SOURCE, "webWorkers": web_workers(), "primary": is_primary(),
|
| 1263 |
"syncRuntime": sync_runtime.runtime_name()[0],
|
| 1264 |
"tenant": writer_tenant(), "scope": worker_scope(), "tenantWorkers": fleet,
|
| 1265 |
"tenantWorkersOverCap": list(_FLEET_STATE["overCap"]), "syncWorkersMax": sync_worker_cap(),
|
| 1266 |
"worker": {"pid": pid, "alive": alive, "heartbeatAgeS": age,
|
| 1267 |
+
"restarts": _WORKER["restarts"], "lastInstall": dict(LAST_INSTALL) or None,
|
| 1268 |
+
"lastInstallError": dict(LAST_INSTALL_ERROR) or None,
|
| 1269 |
"attestation": worker_provenance.visible(stored) if stored.get("heartbeat") == beat else None},
|
| 1270 |
"writer": {"armed": _WRITER["armed"], "held": _WRITER_HELD.is_set(), "holder": _WRITER["holder"]}}
|
api/routes_amazon_projects.py
CHANGED
|
@@ -38,7 +38,7 @@ FIELDS = {"name": "name", "status": "status", "priority": "priority", "owner": "
|
|
| 38 |
"completedOn": "completed_on", "resultConfigs": "result_configs", "contributors": "contributors"}
|
| 39 |
#: Project JSON keys a client may echo back. They are ignored, never written.
|
| 40 |
READ_ONLY = frozenset(("id", "health", "targetCount", "createdAt", "updatedAt", "createdBy",
|
| 41 |
-
"rollup", "impact", "via", "figures"
|
| 42 |
|
| 43 |
|
| 44 |
def _call(fn, *args, **kwargs):
|
|
@@ -116,7 +116,6 @@ def _project(record, today):
|
|
| 116 |
impact = dict(record.get("impact") or {})
|
| 117 |
impact.pop(projects.RESULT_CONFIGS_KEY, None)
|
| 118 |
return {"id": record["id"], "name": record["name"], "status": record["status"],
|
| 119 |
-
"type": record.get("type", "project"), "listed": record.get("listed", True),
|
| 120 |
"priority": record["priority"], "owner": identity["name"] if identity else record["owner"],
|
| 121 |
"ownerUsername": identity["username"] if identity else None, "startOn": record["start_on"],
|
| 122 |
"targetOn": record["target_on"], "progress": record["progress"], "notes": record["notes"],
|
|
@@ -268,7 +267,7 @@ def _row_ids(cells):
|
|
| 268 |
return sorted(cells)
|
| 269 |
|
| 270 |
|
| 271 |
-
def _write_cells(st, table_key, cells
|
| 272 |
import core.user_tables as ut
|
| 273 |
|
| 274 |
def _set(cur):
|
|
@@ -284,7 +283,7 @@ def _write_cells(st, table_key, cells, *, by=amazon_profit.WRITER, via=amazon_pr
|
|
| 284 |
row[key] = value
|
| 285 |
return cur
|
| 286 |
|
| 287 |
-
ut.update_with_activity(table_key, _set, st=st, by=
|
| 288 |
row_ids=_row_ids(cells))
|
| 289 |
|
| 290 |
|
|
@@ -875,11 +874,6 @@ def edit_project(project_id: str, body: Any = Body(default=None), session: Sessi
|
|
| 875 |
def delete_project(project_id: str, session: Session = Depends(require_session)):
|
| 876 |
"""Delete one project and its targets."""
|
| 877 |
runtime = _writer(session)
|
| 878 |
-
record = _call(projects.get, runtime, project_id)
|
| 879 |
-
if record.get("type") == "pdp":
|
| 880 |
-
from routes_amazon_pdp import delete_workflow
|
| 881 |
-
delete_workflow(project_id, session=session)
|
| 882 |
-
return {"ok": True, "id": project_id}
|
| 883 |
pairs = _target_pairs(runtime, project_id)
|
| 884 |
_call(projects.delete, runtime, project_id, by=_by(session))
|
| 885 |
_repaint(runtime, pairs)
|
|
|
|
| 38 |
"completedOn": "completed_on", "resultConfigs": "result_configs", "contributors": "contributors"}
|
| 39 |
#: Project JSON keys a client may echo back. They are ignored, never written.
|
| 40 |
READ_ONLY = frozenset(("id", "health", "targetCount", "createdAt", "updatedAt", "createdBy",
|
| 41 |
+
"rollup", "impact", "via", "figures"))
|
| 42 |
|
| 43 |
|
| 44 |
def _call(fn, *args, **kwargs):
|
|
|
|
| 116 |
impact = dict(record.get("impact") or {})
|
| 117 |
impact.pop(projects.RESULT_CONFIGS_KEY, None)
|
| 118 |
return {"id": record["id"], "name": record["name"], "status": record["status"],
|
|
|
|
| 119 |
"priority": record["priority"], "owner": identity["name"] if identity else record["owner"],
|
| 120 |
"ownerUsername": identity["username"] if identity else None, "startOn": record["start_on"],
|
| 121 |
"targetOn": record["target_on"], "progress": record["progress"], "notes": record["notes"],
|
|
|
|
| 267 |
return sorted(cells)
|
| 268 |
|
| 269 |
|
| 270 |
+
def _write_cells(st, table_key, cells):
|
| 271 |
import core.user_tables as ut
|
| 272 |
|
| 273 |
def _set(cur):
|
|
|
|
| 283 |
row[key] = value
|
| 284 |
return cur
|
| 285 |
|
| 286 |
+
ut.update_with_activity(table_key, _set, st=st, by=amazon_profit.WRITER, via=amazon_profit.VIA,
|
| 287 |
row_ids=_row_ids(cells))
|
| 288 |
|
| 289 |
|
|
|
|
| 874 |
def delete_project(project_id: str, session: Session = Depends(require_session)):
|
| 875 |
"""Delete one project and its targets."""
|
| 876 |
runtime = _writer(session)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 877 |
pairs = _target_pairs(runtime, project_id)
|
| 878 |
_call(projects.delete, runtime, project_id, by=_by(session))
|
| 879 |
_repaint(runtime, pairs)
|
api/routes_brand.py
CHANGED
|
@@ -35,16 +35,13 @@ from __future__ import annotations
|
|
| 35 |
|
| 36 |
import os
|
| 37 |
import re
|
| 38 |
-
from hashlib import sha256
|
| 39 |
from xml.etree import ElementTree
|
| 40 |
from pathlib import Path
|
| 41 |
|
| 42 |
-
from fastapi import APIRouter, Depends
|
| 43 |
-
from fastapi.
|
| 44 |
-
from fastapi.responses import FileResponse, Response
|
| 45 |
|
| 46 |
from deps import Session, err, require_session
|
| 47 |
-
from core import tenant_brand
|
| 48 |
|
| 49 |
router = APIRouter(prefix="/api/v1")
|
| 50 |
|
|
@@ -80,53 +77,6 @@ _MEDIA = {".png": "image/png", ".svg": "image/svg+xml"}
|
|
| 80 |
_HERE = Path(__file__).resolve().parent
|
| 81 |
|
| 82 |
|
| 83 |
-
def _logo_call(fn, *args):
|
| 84 |
-
try:
|
| 85 |
-
return fn(*args)
|
| 86 |
-
except tenant_brand.LogoStorageError as error:
|
| 87 |
-
raise err(503, "logo_storage_unavailable", str(error)) from None
|
| 88 |
-
except ValueError as error:
|
| 89 |
-
raise err(422, "invalid_logo", str(error)) from None
|
| 90 |
-
|
| 91 |
-
|
| 92 |
-
def _logo_admin(session):
|
| 93 |
-
if not session.admin:
|
| 94 |
-
raise err(403, "admin_required", "Only an administrator can change the tenant logo.")
|
| 95 |
-
|
| 96 |
-
|
| 97 |
-
@router.get("/tenant-brand/logo")
|
| 98 |
-
def tenant_logo(session: Session = Depends(require_session)):
|
| 99 |
-
"""Serve this tenant's logo to any signed-in user without changing state."""
|
| 100 |
-
result = _logo_call(tenant_brand.logo, session.runtime)
|
| 101 |
-
if result is None:
|
| 102 |
-
raise err(404, "no_logo", "This workspace has no logo.")
|
| 103 |
-
data, mime = result
|
| 104 |
-
return Response(data, media_type=mime, headers={
|
| 105 |
-
"ETag": f'"{sha256(data).hexdigest()}"', "Cache-Control": "private, no-cache",
|
| 106 |
-
"X-Content-Type-Options": "nosniff"})
|
| 107 |
-
|
| 108 |
-
|
| 109 |
-
@router.put("/tenant-brand/logo")
|
| 110 |
-
async def put_tenant_logo(request: Request, session: Session = Depends(require_session)):
|
| 111 |
-
"""Accept at most 2 MiB of raw PNG or JPEG bytes from an administrator."""
|
| 112 |
-
_logo_admin(session)
|
| 113 |
-
data = bytearray()
|
| 114 |
-
async for chunk in request.stream():
|
| 115 |
-
if len(data) + len(chunk) > tenant_brand.MAX_BYTES:
|
| 116 |
-
raise err(413, "logo_too_large", tenant_brand.SIZE_LIMIT)
|
| 117 |
-
data.extend(chunk)
|
| 118 |
-
return await run_in_threadpool(
|
| 119 |
-
_logo_call, tenant_brand.put_logo, session.runtime, bytes(data), session.uname)
|
| 120 |
-
|
| 121 |
-
|
| 122 |
-
@router.delete("/tenant-brand/logo", status_code=204)
|
| 123 |
-
def delete_tenant_logo(session: Session = Depends(require_session)):
|
| 124 |
-
"""Remove this tenant's logo for an administrator."""
|
| 125 |
-
_logo_admin(session)
|
| 126 |
-
_logo_call(tenant_brand.delete_logo, session.runtime)
|
| 127 |
-
return Response(status_code=204)
|
| 128 |
-
|
| 129 |
-
|
| 130 |
def _roots():
|
| 131 |
env = (os.environ.get("AIOS_BRAND_DIR") or "").strip()
|
| 132 |
out = [Path(env)] if env else []
|
|
|
|
| 35 |
|
| 36 |
import os
|
| 37 |
import re
|
|
|
|
| 38 |
from xml.etree import ElementTree
|
| 39 |
from pathlib import Path
|
| 40 |
|
| 41 |
+
from fastapi import APIRouter, Depends
|
| 42 |
+
from fastapi.responses import FileResponse
|
|
|
|
| 43 |
|
| 44 |
from deps import Session, err, require_session
|
|
|
|
| 45 |
|
| 46 |
router = APIRouter(prefix="/api/v1")
|
| 47 |
|
|
|
|
| 77 |
_HERE = Path(__file__).resolve().parent
|
| 78 |
|
| 79 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 80 |
def _roots():
|
| 81 |
env = (os.environ.get("AIOS_BRAND_DIR") or "").strip()
|
| 82 |
out = [Path(env)] if env else []
|
api/routes_changes.py
CHANGED
|
@@ -62,7 +62,6 @@ import core.store as store
|
|
| 62 |
import core.user_tables as user_tables
|
| 63 |
import modules.customer_data as customer_data
|
| 64 |
import modules.product_data as product_data
|
| 65 |
-
import virtual_sources
|
| 66 |
|
| 67 |
from deps import Session, err, require_changes_session, require_session
|
| 68 |
|
|
@@ -201,8 +200,6 @@ def _may_watch(session, scope):
|
|
| 201 |
@router.get("/changes")
|
| 202 |
def changes(scope: str = "", session: Session | None = Depends(require_changes_session)):
|
| 203 |
"""Serve legacy change checks without making Pg a resident client."""
|
| 204 |
-
if virtual_sources.get(_clean_scope(scope)) is not None:
|
| 205 |
-
return {"scope": _clean_scope(scope), "tokens": {}, "backend": store.backend()}
|
| 206 |
names = buckets_for(scope)
|
| 207 |
if names is None:
|
| 208 |
raise err(400, "bad_scope", "name the table you are watching, e.g. ?scope=customer")
|
|
|
|
| 62 |
import core.user_tables as user_tables
|
| 63 |
import modules.customer_data as customer_data
|
| 64 |
import modules.product_data as product_data
|
|
|
|
| 65 |
|
| 66 |
from deps import Session, err, require_changes_session, require_session
|
| 67 |
|
|
|
|
| 200 |
@router.get("/changes")
|
| 201 |
def changes(scope: str = "", session: Session | None = Depends(require_changes_session)):
|
| 202 |
"""Serve legacy change checks without making Pg a resident client."""
|
|
|
|
|
|
|
| 203 |
names = buckets_for(scope)
|
| 204 |
if names is None:
|
| 205 |
raise err(400, "bad_scope", "name the table you are watching, e.g. ?scope=customer")
|
api/routes_competitors.py
CHANGED
|
@@ -39,19 +39,6 @@ def _open_mirror(runtime):
|
|
| 39 |
cur.close()
|
| 40 |
|
| 41 |
|
| 42 |
-
@contextmanager
|
| 43 |
-
def _boundary_errors():
|
| 44 |
-
try:
|
| 45 |
-
yield
|
| 46 |
-
except rivals.MirrorUnreadable as exc:
|
| 47 |
-
raise err(503, "amazon_mirror_unreadable",
|
| 48 |
-
f"The Amazon data could not be read ({exc}). Try again shortly.") from None
|
| 49 |
-
except rivals.BackendUnavailable:
|
| 50 |
-
raise err(503, "competitors_store_unavailable", "Competitors need the workspace database.") from None
|
| 51 |
-
except RuntimeError:
|
| 52 |
-
raise err(503, "store_not_ready", "Amazon data is still loading. Try again shortly.") from None
|
| 53 |
-
|
| 54 |
-
|
| 55 |
def _rt(session):
|
| 56 |
session.require(routes_amazon_profit.amazon_profit.TABLE_KEY)
|
| 57 |
try:
|
|
@@ -164,23 +151,6 @@ def competitors_list(grain: str, key: str, session: Session = Depends(require_se
|
|
| 164 |
return _payload(runtime, grain, key, target, parent, routes_amazon_profit._may_edit(session), problem)
|
| 165 |
|
| 166 |
|
| 167 |
-
@router.get("/competitors/pool")
|
| 168 |
-
def competitors_pool(grain: str, key: str, session: Session = Depends(require_session)):
|
| 169 |
-
"""Read the eligible Setup pool and its stored daily facts through the caller's tenant session."""
|
| 170 |
-
runtime = _rt(session)
|
| 171 |
-
key = _grain(grain, key)
|
| 172 |
-
with _boundary_errors(), _open_mirror(runtime) as cur:
|
| 173 |
-
target, parent = _target(cur, grain, key)
|
| 174 |
-
if parent is not None:
|
| 175 |
-
raise err(400, "parent_has_no_rivals", rivals.PARENT_ROLLS_UP)
|
| 176 |
-
pool = rivals.pool(runtime, cur, target)
|
| 177 |
-
payload = {field: pool[field] for field in ('asin', 'productType', 'dataAsOf', 'week', 'month',
|
| 178 |
-
'keywordRanksOn', 'cause', 'held', 'removed', 'competitors')}
|
| 179 |
-
if not payload['competitors']:
|
| 180 |
-
payload['cause'] = rivals.NO_POOL
|
| 181 |
-
return {**payload, 'canEdit': routes_amazon_profit._may_edit(session)}
|
| 182 |
-
|
| 183 |
-
|
| 184 |
def _writable(session, body):
|
| 185 |
runtime = _rt(session)
|
| 186 |
if not routes_amazon_profit._may_edit(session):
|
|
@@ -188,8 +158,16 @@ def _writable(session, body):
|
|
| 188 |
body = body if isinstance(body, dict) else {}
|
| 189 |
grain = body.get("grain")
|
| 190 |
key = _grain(grain, body.get("key"))
|
| 191 |
-
|
| 192 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 193 |
if parent is not None:
|
| 194 |
raise err(400, "parent_has_no_rivals", rivals.PARENT_ROLLS_UP)
|
| 195 |
try:
|
|
|
|
| 39 |
cur.close()
|
| 40 |
|
| 41 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 42 |
def _rt(session):
|
| 43 |
session.require(routes_amazon_profit.amazon_profit.TABLE_KEY)
|
| 44 |
try:
|
|
|
|
| 151 |
return _payload(runtime, grain, key, target, parent, routes_amazon_profit._may_edit(session), problem)
|
| 152 |
|
| 153 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 154 |
def _writable(session, body):
|
| 155 |
runtime = _rt(session)
|
| 156 |
if not routes_amazon_profit._may_edit(session):
|
|
|
|
| 158 |
body = body if isinstance(body, dict) else {}
|
| 159 |
grain = body.get("grain")
|
| 160 |
key = _grain(grain, body.get("key"))
|
| 161 |
+
try:
|
| 162 |
+
with _open_mirror(runtime) as cur:
|
| 163 |
+
target, parent = _target(cur, grain, key)
|
| 164 |
+
except rivals.MirrorUnreadable as exc:
|
| 165 |
+
raise err(503, "amazon_mirror_unreadable",
|
| 166 |
+
f"The Amazon data could not be read ({exc}). Try again shortly.") from None
|
| 167 |
+
except rivals.BackendUnavailable:
|
| 168 |
+
raise err(503, "competitors_store_unavailable", "Competitors need the workspace database.") from None
|
| 169 |
+
except RuntimeError:
|
| 170 |
+
raise err(503, "store_not_ready", "Amazon data is still loading. Try again shortly.") from None
|
| 171 |
if parent is not None:
|
| 172 |
raise err(400, "parent_has_no_rivals", rivals.PARENT_ROLLS_UP)
|
| 173 |
try:
|
api/routes_connectors.py
CHANGED
|
@@ -3315,6 +3315,9 @@ def _amazon_table_schedule(rt, connection_id=None, now=None):
|
|
| 3315 |
state = (landed.get("state") if landed.get("state") in ("refused", "failed", "throttled")
|
| 3316 |
else tick.get("state") or "never_run")
|
| 3317 |
reason = row_reason(table, state)
|
|
|
|
|
|
|
|
|
|
| 3318 |
# A source-internal mismatch published the table. Its warning stays until a clean reconcile.
|
| 3319 |
warning = _amazon_cause_text(saved.get(prefix + az.entity_key("warning_" + table)) or "")
|
| 3320 |
schedule.append({**row, "period": _amazon_table_period(table),
|
|
@@ -3354,7 +3357,9 @@ def _amazon_sync_staleness(completed, schedule, now=None):
|
|
| 3354 |
fastest = max(amazon_client.CADENCE[table]["perDay"]
|
| 3355 |
for table in amazon_client.GROUP_A)
|
| 3356 |
allowed = 86400 / fastest + rel.STALE_GRACE_SECONDS
|
| 3357 |
-
|
|
|
|
|
|
|
| 3358 |
if not stamps:
|
| 3359 |
stamps = [completed]
|
| 3360 |
now = now or datetime.now(timezone.utc)
|
|
|
|
| 3315 |
state = (landed.get("state") if landed.get("state") in ("refused", "failed", "throttled")
|
| 3316 |
else tick.get("state") or "never_run")
|
| 3317 |
reason = row_reason(table, state)
|
| 3318 |
+
if state == "refused" and (reason or "").startswith("Amazon does not offer this report"):
|
| 3319 |
+
# A report Amazon does not offer this seller is not available, never a failed sync.
|
| 3320 |
+
state = "not_offered"
|
| 3321 |
# A source-internal mismatch published the table. Its warning stays until a clean reconcile.
|
| 3322 |
warning = _amazon_cause_text(saved.get(prefix + az.entity_key("warning_" + table)) or "")
|
| 3323 |
schedule.append({**row, "period": _amazon_table_period(table),
|
|
|
|
| 3357 |
fastest = max(amazon_client.CADENCE[table]["perDay"]
|
| 3358 |
for table in amazon_client.GROUP_A)
|
| 3359 |
allowed = 86400 / fastest + rel.STALE_GRACE_SECONDS
|
| 3360 |
+
# A not_offered row can carry an old watermark stamp. Amazon never sends that report, so it never ages the card.
|
| 3361 |
+
stamps = [row.get("lastSyncAt") for row in schedule
|
| 3362 |
+
if row["perDay"] == fastest and row.get("state") != "not_offered"]
|
| 3363 |
if not stamps:
|
| 3364 |
stamps = [completed]
|
| 3365 |
now = now or datetime.now(timezone.utc)
|
api/routes_grid.py
CHANGED
|
@@ -20,7 +20,6 @@ from inspect import signature
|
|
| 20 |
from fastapi import APIRouter, Body, Depends, Header, HTTPException, Query
|
| 21 |
|
| 22 |
from deps import Session, err, module_gate, perms, require_session
|
| 23 |
-
import virtual_sources
|
| 24 |
|
| 25 |
router = APIRouter(prefix="/api/v1")
|
| 26 |
|
|
@@ -50,7 +49,6 @@ def row_patch_receipt(scope):
|
|
| 50 |
raise err(400, 'bad_op_id', str(exc)) from exc
|
| 51 |
session = bound.arguments['session']
|
| 52 |
scope_key = scope if isinstance(scope, str) else scope(bound.arguments)
|
| 53 |
-
virtual_sources.refuse_write(scope_key)
|
| 54 |
bound.arguments['body'] = {key: value for key, value in body.items() if key != 'opId'}
|
| 55 |
|
| 56 |
def apply():
|
|
@@ -235,7 +233,6 @@ def grid_choice_remove(scope: str = "customer", body: dict = Body(default=None),
|
|
| 235 |
|
| 236 |
body = body if isinstance(body, dict) else {}
|
| 237 |
scope = _scope_or_400(scope)
|
| 238 |
-
virtual_sources.refuse_write(scope)
|
| 239 |
key = str(body.get("key") or "").strip()[:80]
|
| 240 |
options = _cr.normalise(body.get("options") if isinstance(body.get("options"), list) else [])
|
| 241 |
confirm = body.get("confirm") is True
|
|
@@ -478,8 +475,6 @@ def workspace(scope: str = "customer",
|
|
| 478 |
from routes_customers import grid_assembly
|
| 479 |
|
| 480 |
scope = _scope_or_400(scope)
|
| 481 |
-
if virtual_sources.get(scope) is not None:
|
| 482 |
-
return {"workspace": virtual_sources.assembly(session, scope, with_rows=False)["workspace"]}
|
| 483 |
# Wave 21 C4 — the per-scope gate the dependency no longer asserts: customer/cohort need
|
| 484 |
# the customer module; product asserts PRODUCT_MODULE in its branch; ut_* needs only a
|
| 485 |
# session (its wall is the table's own, inside the assembly).
|
|
@@ -1529,7 +1524,6 @@ def bulk_cells(body: dict = Body(default=None),
|
|
| 1529 |
from routes_products import MODULE as PRODUCT_MODULE, scoped_pool
|
| 1530 |
|
| 1531 |
scope = _scope_or_400((body or {}).get("scopeKey"))
|
| 1532 |
-
virtual_sources.refuse_write(scope)
|
| 1533 |
if scope != "product":
|
| 1534 |
# Fail-closed with the reason: the shared stratum is a PRODUCT-topic mechanism today.
|
| 1535 |
raise err(400, "scope_not_bulk_writable",
|
|
@@ -2132,7 +2126,6 @@ def grid_events_route(body: dict = Body(default=None),
|
|
| 2132 |
# typo degraded silently to customer-scope behaviour on a WRITE. Read and write must agree on
|
| 2133 |
# what a scope is, or the surface you read is not the surface you wrote.
|
| 2134 |
scope = _scope_or_400((body or {}).get("scopeKey"))
|
| 2135 |
-
virtual_sources.refuse_write(scope)
|
| 2136 |
if scope.startswith("ut_") and any(
|
| 2137 |
isinstance(event, dict) and event.get("type") == "overlay_patch" for event in events):
|
| 2138 |
# Managed records require Route/Zone tools, including keys without stored definitions.
|
|
|
|
| 20 |
from fastapi import APIRouter, Body, Depends, Header, HTTPException, Query
|
| 21 |
|
| 22 |
from deps import Session, err, module_gate, perms, require_session
|
|
|
|
| 23 |
|
| 24 |
router = APIRouter(prefix="/api/v1")
|
| 25 |
|
|
|
|
| 49 |
raise err(400, 'bad_op_id', str(exc)) from exc
|
| 50 |
session = bound.arguments['session']
|
| 51 |
scope_key = scope if isinstance(scope, str) else scope(bound.arguments)
|
|
|
|
| 52 |
bound.arguments['body'] = {key: value for key, value in body.items() if key != 'opId'}
|
| 53 |
|
| 54 |
def apply():
|
|
|
|
| 233 |
|
| 234 |
body = body if isinstance(body, dict) else {}
|
| 235 |
scope = _scope_or_400(scope)
|
|
|
|
| 236 |
key = str(body.get("key") or "").strip()[:80]
|
| 237 |
options = _cr.normalise(body.get("options") if isinstance(body.get("options"), list) else [])
|
| 238 |
confirm = body.get("confirm") is True
|
|
|
|
| 475 |
from routes_customers import grid_assembly
|
| 476 |
|
| 477 |
scope = _scope_or_400(scope)
|
|
|
|
|
|
|
| 478 |
# Wave 21 C4 — the per-scope gate the dependency no longer asserts: customer/cohort need
|
| 479 |
# the customer module; product asserts PRODUCT_MODULE in its branch; ut_* needs only a
|
| 480 |
# session (its wall is the table's own, inside the assembly).
|
|
|
|
| 1524 |
from routes_products import MODULE as PRODUCT_MODULE, scoped_pool
|
| 1525 |
|
| 1526 |
scope = _scope_or_400((body or {}).get("scopeKey"))
|
|
|
|
| 1527 |
if scope != "product":
|
| 1528 |
# Fail-closed with the reason: the shared stratum is a PRODUCT-topic mechanism today.
|
| 1529 |
raise err(400, "scope_not_bulk_writable",
|
|
|
|
| 2126 |
# typo degraded silently to customer-scope behaviour on a WRITE. Read and write must agree on
|
| 2127 |
# what a scope is, or the surface you read is not the surface you wrote.
|
| 2128 |
scope = _scope_or_400((body or {}).get("scopeKey"))
|
|
|
|
| 2129 |
if scope.startswith("ut_") and any(
|
| 2130 |
isinstance(event, dict) and event.get("type") == "overlay_patch" for event in events):
|
| 2131 |
# Managed records require Route/Zone tools, including keys without stored definitions.
|
api/routes_odoo_tables.py
CHANGED
|
@@ -5096,12 +5096,10 @@ def _materialised_window(table_key, session, offset, limit, filters, filterConj,
|
|
| 5096 |
def _materialised_window_read(table_key, session, offset, limit, filters, filterConj, sorts, search, related,
|
| 5097 |
groupBy=None, groupKey=None, ids=None, cursor=None):
|
| 5098 |
from harness import datastore
|
| 5099 |
-
import virtual_sources
|
| 5100 |
|
| 5101 |
query = _selection_query({"filters": _json_arg(filters, "filters"), "filterConj": filterConj,
|
| 5102 |
"sorts": _json_arg(sorts, "sorts"), "search": search})
|
| 5103 |
-
|
| 5104 |
-
payload = virtual_sources.rows_payload(session, table_key, related=related) if virtual else None
|
| 5105 |
window_offset = offset
|
| 5106 |
if table_key == 'ut_activity':
|
| 5107 |
if cursor is not None:
|
|
@@ -5140,9 +5138,7 @@ def _materialised_window_read(table_key, session, offset, limit, filters, filter
|
|
| 5140 |
options = dict(select=spec['id'] if read.get('selection') else '_row_index', where=where, params=tuple(params),
|
| 5141 |
order_by=order or spec['id'], offset=window_offset, limit=limit or None,
|
| 5142 |
cur=read['cur'], **read['frm'])
|
| 5143 |
-
if table_key == 'ut_activity'
|
| 5144 |
-
if virtual and cursor is not None:
|
| 5145 |
-
raise err(400, 'window_cursor_unsupported', 'Continue this source with an offset.')
|
| 5146 |
win = datastore.window(**options)
|
| 5147 |
win['next_cursor'] = (read['payload'] or {}).get('next_cursor')
|
| 5148 |
else:
|
|
@@ -5289,10 +5285,6 @@ def odoo_table_rows(table_key: str,
|
|
| 5289 |
"limit, filters, sorts and search in the query string.")
|
| 5290 |
ids = _record_ids(body)
|
| 5291 |
|
| 5292 |
-
import virtual_sources
|
| 5293 |
-
if virtual_sources.get(table_key) is not None:
|
| 5294 |
-
return _materialised_window_read(table_key, session, offset, limit, filters, filterConj,
|
| 5295 |
-
sorts, search, related, groupBy, groupKey, ids=ids, cursor=cursor)
|
| 5296 |
# Admission resolves this Amazon database without publishing a process-wide binding.
|
| 5297 |
page = _page_defs(session) if str(table_key).startswith(("ut_amzn_", "ut_odoo_model_")) else None
|
| 5298 |
admitted = require_amazon_mirror(table_key, session, st=page[0] if page else None)
|
|
|
|
| 5096 |
def _materialised_window_read(table_key, session, offset, limit, filters, filterConj, sorts, search, related,
|
| 5097 |
groupBy=None, groupKey=None, ids=None, cursor=None):
|
| 5098 |
from harness import datastore
|
|
|
|
| 5099 |
|
| 5100 |
query = _selection_query({"filters": _json_arg(filters, "filters"), "filterConj": filterConj,
|
| 5101 |
"sorts": _json_arg(sorts, "sorts"), "search": search})
|
| 5102 |
+
payload = None
|
|
|
|
| 5103 |
window_offset = offset
|
| 5104 |
if table_key == 'ut_activity':
|
| 5105 |
if cursor is not None:
|
|
|
|
| 5138 |
options = dict(select=spec['id'] if read.get('selection') else '_row_index', where=where, params=tuple(params),
|
| 5139 |
order_by=order or spec['id'], offset=window_offset, limit=limit or None,
|
| 5140 |
cur=read['cur'], **read['frm'])
|
| 5141 |
+
if table_key == 'ut_activity':
|
|
|
|
|
|
|
| 5142 |
win = datastore.window(**options)
|
| 5143 |
win['next_cursor'] = (read['payload'] or {}).get('next_cursor')
|
| 5144 |
else:
|
|
|
|
| 5285 |
"limit, filters, sorts and search in the query string.")
|
| 5286 |
ids = _record_ids(body)
|
| 5287 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5288 |
# Admission resolves this Amazon database without publishing a process-wide binding.
|
| 5289 |
page = _page_defs(session) if str(table_key).startswith(("ut_amzn_", "ut_odoo_model_")) else None
|
| 5290 |
admitted = require_amazon_mirror(table_key, session, st=page[0] if page else None)
|
api/routes_shares.py
CHANGED
|
@@ -1221,8 +1221,6 @@ def get_cohort(scope: str, cohort_id: str, session: Session = Depends(require_se
|
|
| 1221 |
@router.get("/share/{kind}/{oid}")
|
| 1222 |
def get_share(kind: str, oid: str, session: Session = Depends(require_session)):
|
| 1223 |
kind = _kind_or_400(kind)
|
| 1224 |
-
if kind == "report" and not session.admin:
|
| 1225 |
-
raise err(404, "no_object", "no such item, or it is not shared with this account")
|
| 1226 |
if kind == "field":
|
| 1227 |
table_key, field_key = shares.split_field_oid(oid)
|
| 1228 |
if table_key and field_key:
|
|
@@ -1281,9 +1279,8 @@ def get_share(kind: str, oid: str, session: Session = Depends(require_session)):
|
|
| 1281 |
**rec,
|
| 1282 |
"role": role,
|
| 1283 |
"mayAdminister": may_admin,
|
| 1284 |
-
"maxGrantableRole": (
|
| 1285 |
-
|
| 1286 |
-
kind, oid, session.uname, is_admin=_admin_share, st=session.runtime)),
|
| 1287 |
"ownerLabel": owner_label,
|
| 1288 |
# ⚠ WAVE 21 (C1 identity fix): grant entries BIND on USERNAMES, so the picker must carry
|
| 1289 |
# them. `assignable_people` serves bare display names because `user`-kind CELLS store
|
|
@@ -1305,19 +1302,35 @@ def get_share(kind: str, oid: str, session: Session = Depends(require_session)):
|
|
| 1305 |
|
| 1306 |
|
| 1307 |
def _people(tenant):
|
| 1308 |
-
"""
|
| 1309 |
-
|
| 1310 |
-
|
| 1311 |
-
|
| 1312 |
-
|
| 1313 |
-
|
| 1314 |
-
|
| 1315 |
-
|
| 1316 |
-
|
| 1317 |
-
|
| 1318 |
-
|
| 1319 |
-
|
| 1320 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1321 |
|
| 1322 |
|
| 1323 |
def _stranger(session, kind, oid):
|
|
@@ -1342,11 +1355,6 @@ def _put_share(kind, oid, body, session, *, _view_snapshot=None, _source_owner=N
|
|
| 1342 |
_notify=True):
|
| 1343 |
"""Apply the shared share admission, ceiling and atomic writer for both doors."""
|
| 1344 |
kind = _kind_or_400(kind)
|
| 1345 |
-
if kind == "report" and not session.admin:
|
| 1346 |
-
if _stranger(session, kind, oid):
|
| 1347 |
-
raise err(404, "no_object", "no such item, or it is not shared with this account")
|
| 1348 |
-
raise err(403, "not_owner",
|
| 1349 |
-
"Only an administrator can change who can download this report.")
|
| 1350 |
if kind == "field":
|
| 1351 |
table_key, field_key = shares.split_field_oid(oid)
|
| 1352 |
if table_key and field_key:
|
|
@@ -1420,8 +1428,6 @@ def _put_share(kind, oid, body, session, *, _view_snapshot=None, _source_owner=N
|
|
| 1420 |
raise err(400, "bad_entries",
|
| 1421 |
"entries must be a list of {user, role}. Send [] to un-share, which is how "
|
| 1422 |
"revoking is expressed")
|
| 1423 |
-
if kind == "report":
|
| 1424 |
-
_report_entries_or_400(entries)
|
| 1425 |
_entries_or_400(session, entries)
|
| 1426 |
# ⭐⭐ R4 / W40-T02 — THE CEILING. `may_administer` above now opens this door to an `edit`
|
| 1427 |
# grantee on a VIEW, so R4's other half ("a re-share may never exceed the role the re-sharer
|
|
@@ -1617,8 +1623,7 @@ def _put_share(kind, oid, body, session, *, _view_snapshot=None, _source_owner=N
|
|
| 1617 |
|
| 1618 |
try:
|
| 1619 |
out = shares.set_grants(kind, oid, entries,
|
| 1620 |
-
owner=
|
| 1621 |
-
session.uname if claiming else rec["owner"]),
|
| 1622 |
granter=session.uname,
|
| 1623 |
st=session.runtime,
|
| 1624 |
expect=rec["entries"])
|
|
|
|
| 1221 |
@router.get("/share/{kind}/{oid}")
|
| 1222 |
def get_share(kind: str, oid: str, session: Session = Depends(require_session)):
|
| 1223 |
kind = _kind_or_400(kind)
|
|
|
|
|
|
|
| 1224 |
if kind == "field":
|
| 1225 |
table_key, field_key = shares.split_field_oid(oid)
|
| 1226 |
if table_key and field_key:
|
|
|
|
| 1279 |
**rec,
|
| 1280 |
"role": role,
|
| 1281 |
"mayAdminister": may_admin,
|
| 1282 |
+
"maxGrantableRole": "edit" if claiming else shares.max_grantable_role(
|
| 1283 |
+
kind, oid, session.uname, is_admin=_admin_share, st=session.runtime),
|
|
|
|
| 1284 |
"ownerLabel": owner_label,
|
| 1285 |
# ⚠ WAVE 21 (C1 identity fix): grant entries BIND on USERNAMES, so the picker must carry
|
| 1286 |
# them. `assignable_people` serves bare display names because `user`-kind CELLS store
|
|
|
|
| 1302 |
|
| 1303 |
|
| 1304 |
def _people(tenant):
|
| 1305 |
+
"""[{username, name}] for this tenant — same population as `assignable_people`, with the
|
| 1306 |
+
BINDING identity alongside the display one.
|
| 1307 |
+
|
| 1308 |
+
⛔⛔ W49-T23 / D-235 — A BLANK TENANT RETURNS NOBODY, NOT EVERYBODY. This read
|
| 1309 |
+
`if want and str(u.get('tenant') …) != want`, so a blank or `None` slug skipped the filter
|
| 1310 |
+
entirely and handed one workspace every tenant's roster: the exact `if want and …` fail-OPEN
|
| 1311 |
+
that W33-T37 closed in `core.users.assignable_people` / `avatar_map` one layer down, left
|
| 1312 |
+
behind in the copy that lives here. The three callers all pass `session.tenant`, which is
|
| 1313 |
+
`claims["t"]` and is already proven non-blank by `require_session` (an unresolvable slug is a
|
| 1314 |
+
401 before any route runs), so this changes no answer any live caller gets today — it removes
|
| 1315 |
+
the shape, before a fourth caller arrives that does not carry that proof. Fail-closed, because
|
| 1316 |
+
from this function a blank slug and "every tenant at once" are the same observation, and one
|
| 1317 |
+
of the two callers below turns this roster into who may be GRANTED access.
|
| 1318 |
+
"""
|
| 1319 |
+
try:
|
| 1320 |
+
reg = users.registry() or {}
|
| 1321 |
+
except Exception:
|
| 1322 |
+
return []
|
| 1323 |
+
want = str(tenant or '').strip().lower()
|
| 1324 |
+
if not want:
|
| 1325 |
+
return []
|
| 1326 |
+
out = []
|
| 1327 |
+
for uname, u in reg.items():
|
| 1328 |
+
if not isinstance(u, dict) or u.get('active') is False:
|
| 1329 |
+
continue
|
| 1330 |
+
if str(u.get('tenant') or 'royal-imports').strip().lower() != want:
|
| 1331 |
+
continue
|
| 1332 |
+
out.append({"username": str(uname), "name": str(u.get('name') or uname)})
|
| 1333 |
+
return sorted(out, key=lambda p: p["name"].lower())
|
| 1334 |
|
| 1335 |
|
| 1336 |
def _stranger(session, kind, oid):
|
|
|
|
| 1355 |
_notify=True):
|
| 1356 |
"""Apply the shared share admission, ceiling and atomic writer for both doors."""
|
| 1357 |
kind = _kind_or_400(kind)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1358 |
if kind == "field":
|
| 1359 |
table_key, field_key = shares.split_field_oid(oid)
|
| 1360 |
if table_key and field_key:
|
|
|
|
| 1428 |
raise err(400, "bad_entries",
|
| 1429 |
"entries must be a list of {user, role}. Send [] to un-share, which is how "
|
| 1430 |
"revoking is expressed")
|
|
|
|
|
|
|
| 1431 |
_entries_or_400(session, entries)
|
| 1432 |
# ⭐⭐ R4 / W40-T02 — THE CEILING. `may_administer` above now opens this door to an `edit`
|
| 1433 |
# grantee on a VIEW, so R4's other half ("a re-share may never exceed the role the re-sharer
|
|
|
|
| 1623 |
|
| 1624 |
try:
|
| 1625 |
out = shares.set_grants(kind, oid, entries,
|
| 1626 |
+
owner=session.uname if claiming else rec["owner"],
|
|
|
|
| 1627 |
granter=session.uname,
|
| 1628 |
st=session.runtime,
|
| 1629 |
expect=rec["entries"])
|
api/routes_starred.py
CHANGED
|
@@ -1231,9 +1231,6 @@ def starred_records(database: str = "", session: Session = Depends(require_sessi
|
|
| 1231 |
key = _clean_id(database)
|
| 1232 |
if not key:
|
| 1233 |
raise err(400, "bad_request", "no database was named")
|
| 1234 |
-
import virtual_sources
|
| 1235 |
-
if virtual_sources.get(key) is not None:
|
| 1236 |
-
return {"database": key, "ids": [], "count": 0, "view": None}
|
| 1237 |
return _records_payload(session, key, _database_or_refuse(session, key))
|
| 1238 |
|
| 1239 |
|
|
@@ -1252,8 +1249,6 @@ def set_starred_record(body: dict = Body(default=None),
|
|
| 1252 |
raw_id = _clean_id(body.get("id"))
|
| 1253 |
if not key:
|
| 1254 |
raise err(400, "bad_request", "no database was named")
|
| 1255 |
-
import virtual_sources
|
| 1256 |
-
virtual_sources.refuse_write(key)
|
| 1257 |
if not raw_id.isdigit():
|
| 1258 |
# Named rather than coerced: see `_record_ids` on why a non-numeric id cannot be stored.
|
| 1259 |
raise err(400, "bad_request", "a record id is a number")
|
|
|
|
| 1231 |
key = _clean_id(database)
|
| 1232 |
if not key:
|
| 1233 |
raise err(400, "bad_request", "no database was named")
|
|
|
|
|
|
|
|
|
|
| 1234 |
return _records_payload(session, key, _database_or_refuse(session, key))
|
| 1235 |
|
| 1236 |
|
|
|
|
| 1249 |
raw_id = _clean_id(body.get("id"))
|
| 1250 |
if not key:
|
| 1251 |
raise err(400, "bad_request", "no database was named")
|
|
|
|
|
|
|
| 1252 |
if not raw_id.isdigit():
|
| 1253 |
# Named rather than coerced: see `_record_ids` on why a non-numeric id cannot be stored.
|
| 1254 |
raise err(400, "bad_request", "a record id is a number")
|
api/routes_tables.py
CHANGED
|
@@ -56,7 +56,6 @@ from fastapi import HTTPException, Request, Response
|
|
| 56 |
from deps import Session, err, require_session
|
| 57 |
from harness.runtime import DatastoreMismatch
|
| 58 |
from routes_grid import row_patch_receipt
|
| 59 |
-
import virtual_sources
|
| 60 |
|
| 61 |
router = APIRouter(prefix="/api/v1")
|
| 62 |
|
|
@@ -286,8 +285,6 @@ def related_metrics(table_key: str, body: dict = Body(default=None),
|
|
| 286 |
if (not isinstance(pin, dict) or not isinstance(pids, list)
|
| 287 |
or any(type(pid) is not int or pid <= 0 for pid in pids)):
|
| 288 |
raise err(400, "bad_metric_scope", "Name the origin record, relation and linked records.")
|
| 289 |
-
if virtual_sources.get(table_key) is not None:
|
| 290 |
-
return virtual_sources.related_metrics(session, table_key, pin, pids)
|
| 291 |
session.require(table_key)
|
| 292 |
viewer = copy.copy(session)
|
| 293 |
viewer.runtime = _MetricReadRuntime(session.runtime)
|
|
@@ -393,8 +390,6 @@ def _table_lend(runtime, table_key, defs_only=False):
|
|
| 393 |
def _managed_or_refuse(session, table_key, *, mutation=False):
|
| 394 |
"""C1 generic-door policy. Dedicated Route/Zone handlers use their own capability."""
|
| 395 |
from core import route_system
|
| 396 |
-
if mutation:
|
| 397 |
-
virtual_sources.refuse_write(table_key)
|
| 398 |
definition = _ut().get_one(table_key, st=session.runtime)
|
| 399 |
# Route/Zone are virtual system descriptors until their first dedicated write, so looking
|
| 400 |
# only in `user_tables` would leave a generic mutation path open during that empty state.
|
|
@@ -589,7 +584,6 @@ def _defn_or_refuse(session, table_key, st=None, defs_only=False, scope_applied=
|
|
| 589 |
proves nothing about which read produced it. `core.store.is_projected(defn)` is how a gate
|
| 590 |
asserts the fast path was TAKEN.
|
| 591 |
"""
|
| 592 |
-
virtual_sources.refuse_write(table_key)
|
| 593 |
ut = _ut()
|
| 594 |
# ⭐⭐ W31 QA — THE SWEEP, AND IT IS ONE LINE BECAUSE IT IS DONE HERE RATHER THAN PER ROUTE.
|
| 595 |
# Owner: *"this is just one case, I need you to check and apply the fix everywhere too."*
|
|
@@ -672,8 +666,8 @@ def _records_lock_message(table_key, definition, st):
|
|
| 672 |
return f"This {name} is locked, so only user-created field values can be edited."
|
| 673 |
|
| 674 |
|
| 675 |
-
def
|
| 676 |
-
"""Require editor authority
|
| 677 |
# Share one snapshot across the read, editor and record-mode checks.
|
| 678 |
st = st if st is not None else _ut().lend(session.runtime)
|
| 679 |
# ⭐ W36-T21 — `scope_applied=True` because every row write behind this wall is bounded by a
|
|
@@ -683,12 +677,6 @@ def _editor_or_refuse(session, table_key, st=None):
|
|
| 683 |
defn = _defn_or_refuse(session, table_key, st=st, scope_applied=True)
|
| 684 |
if not _ut().may_edit_database(table_key, session.uname, is_admin=session.admin, st=st):
|
| 685 |
raise err(403, "database_read_only", "Can edit access is required to change this database.")
|
| 686 |
-
return defn, st
|
| 687 |
-
|
| 688 |
-
|
| 689 |
-
def _records_or_refuse(session, table_key, st=None, *, cell_updates=None):
|
| 690 |
-
"""Require editor authority and reserve locked database edits for authored field values."""
|
| 691 |
-
defn, st = _editor_or_refuse(session, table_key, st)
|
| 692 |
if not _ut().records_mutable(table_key, st=st):
|
| 693 |
if cell_updates and table_key != "ut_activity":
|
| 694 |
from core import shared_overlay
|
|
@@ -1063,9 +1051,6 @@ def _activity_grid_rows(session):
|
|
| 1063 |
@router.get('/tables/{table_key}/activity')
|
| 1064 |
def table_activity(table_key: str, cursor: str = None, limit: int = 50,
|
| 1065 |
session: Session = Depends(require_session)):
|
| 1066 |
-
if virtual_sources.get(table_key) is not None:
|
| 1067 |
-
from core import activity_log
|
| 1068 |
-
return {'records': [], 'next_cursor': None, 'fields': activity_log.definition()['fields']}
|
| 1069 |
if table_key.startswith('ut_'):
|
| 1070 |
_defn_or_refuse(session, table_key, defs_only=True, scope_applied=True)
|
| 1071 |
elif table_key not in ('customer_data', 'product_data'):
|
|
@@ -2137,9 +2122,6 @@ def _scoped_table_rows(session, table_key, st, *, cursor=None, limit=None):
|
|
| 2137 |
def ut_assembly(session: Session, table_key: str, storage_key: str = "",
|
| 2138 |
consume_corrections: bool = False, with_rows: bool = True, st=None,
|
| 2139 |
*, fields_only: bool = False, needed=None, cursor=None, limit=None):
|
| 2140 |
-
if virtual_sources.get(table_key) is not None:
|
| 2141 |
-
return virtual_sources.assembly(session, table_key, with_rows=with_rows and not fields_only,
|
| 2142 |
-
storage_key=storage_key)
|
| 2143 |
from core import grid_events
|
| 2144 |
actor = grid_events.request_admission(session, table_key + '_table_workspace')
|
| 2145 |
result = _ut_assembly(actor, table_key, storage_key, consume_corrections, with_rows, st,
|
|
@@ -3365,8 +3347,6 @@ def table_rows(table_key: str, session: Session = Depends(require_session),
|
|
| 3365 |
(found by this route's own gate check, not by luck)."""
|
| 3366 |
import aios_grid
|
| 3367 |
|
| 3368 |
-
if virtual_sources.get(table_key) is not None:
|
| 3369 |
-
return virtual_sources.rows_payload(session, table_key, related=related)
|
| 3370 |
if table_key == 'ut_activity':
|
| 3371 |
if related:
|
| 3372 |
raise err(409, 'activity_related_unavailable', 'Open the Activity database to read events.')
|
|
@@ -4541,8 +4521,7 @@ def restore_history(table_key: str, event_id: str, body: dict = Body(default=Non
|
|
| 4541 |
from core import grid_events
|
| 4542 |
|
| 4543 |
_managed_or_refuse(session, table_key, mutation=True)
|
| 4544 |
-
|
| 4545 |
-
_editor_or_refuse(session, table_key)
|
| 4546 |
body = body if isinstance(body, dict) else {}
|
| 4547 |
row_id = str(body.get("rowId") or "").strip()
|
| 4548 |
if not row_id:
|
|
@@ -4572,7 +4551,6 @@ def restore_history(table_key: str, event_id: str, body: dict = Body(default=Non
|
|
| 4572 |
raise err(404, "history_event_not_found", "that retained value version is unavailable")
|
| 4573 |
if any(key in (g.get("hidden") or ()) for key in updates):
|
| 4574 |
raise err(403, "field_hidden", "that retained version includes a hidden field")
|
| 4575 |
-
_records_or_refuse(session, table_key, cell_updates=updates)
|
| 4576 |
prior_history_ids = {
|
| 4577 |
str(item.get("id")) for item in _ut().value_history(
|
| 4578 |
table_key, row_id, st=session.runtime, retention_days=days,
|
|
|
|
| 56 |
from deps import Session, err, require_session
|
| 57 |
from harness.runtime import DatastoreMismatch
|
| 58 |
from routes_grid import row_patch_receipt
|
|
|
|
| 59 |
|
| 60 |
router = APIRouter(prefix="/api/v1")
|
| 61 |
|
|
|
|
| 285 |
if (not isinstance(pin, dict) or not isinstance(pids, list)
|
| 286 |
or any(type(pid) is not int or pid <= 0 for pid in pids)):
|
| 287 |
raise err(400, "bad_metric_scope", "Name the origin record, relation and linked records.")
|
|
|
|
|
|
|
| 288 |
session.require(table_key)
|
| 289 |
viewer = copy.copy(session)
|
| 290 |
viewer.runtime = _MetricReadRuntime(session.runtime)
|
|
|
|
| 390 |
def _managed_or_refuse(session, table_key, *, mutation=False):
|
| 391 |
"""C1 generic-door policy. Dedicated Route/Zone handlers use their own capability."""
|
| 392 |
from core import route_system
|
|
|
|
|
|
|
| 393 |
definition = _ut().get_one(table_key, st=session.runtime)
|
| 394 |
# Route/Zone are virtual system descriptors until their first dedicated write, so looking
|
| 395 |
# only in `user_tables` would leave a generic mutation path open during that empty state.
|
|
|
|
| 584 |
proves nothing about which read produced it. `core.store.is_projected(defn)` is how a gate
|
| 585 |
asserts the fast path was TAKEN.
|
| 586 |
"""
|
|
|
|
| 587 |
ut = _ut()
|
| 588 |
# ⭐⭐ W31 QA — THE SWEEP, AND IT IS ONE LINE BECAUSE IT IS DONE HERE RATHER THAN PER ROUTE.
|
| 589 |
# Owner: *"this is just one case, I need you to check and apply the fix everywhere too."*
|
|
|
|
| 666 |
return f"This {name} is locked, so only user-created field values can be edited."
|
| 667 |
|
| 668 |
|
| 669 |
+
def _records_or_refuse(session, table_key, st=None, *, cell_updates=None):
|
| 670 |
+
"""Require editor authority and reserve locked database edits for authored field values."""
|
| 671 |
# Share one snapshot across the read, editor and record-mode checks.
|
| 672 |
st = st if st is not None else _ut().lend(session.runtime)
|
| 673 |
# ⭐ W36-T21 — `scope_applied=True` because every row write behind this wall is bounded by a
|
|
|
|
| 677 |
defn = _defn_or_refuse(session, table_key, st=st, scope_applied=True)
|
| 678 |
if not _ut().may_edit_database(table_key, session.uname, is_admin=session.admin, st=st):
|
| 679 |
raise err(403, "database_read_only", "Can edit access is required to change this database.")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 680 |
if not _ut().records_mutable(table_key, st=st):
|
| 681 |
if cell_updates and table_key != "ut_activity":
|
| 682 |
from core import shared_overlay
|
|
|
|
| 1051 |
@router.get('/tables/{table_key}/activity')
|
| 1052 |
def table_activity(table_key: str, cursor: str = None, limit: int = 50,
|
| 1053 |
session: Session = Depends(require_session)):
|
|
|
|
|
|
|
|
|
|
| 1054 |
if table_key.startswith('ut_'):
|
| 1055 |
_defn_or_refuse(session, table_key, defs_only=True, scope_applied=True)
|
| 1056 |
elif table_key not in ('customer_data', 'product_data'):
|
|
|
|
| 2122 |
def ut_assembly(session: Session, table_key: str, storage_key: str = "",
|
| 2123 |
consume_corrections: bool = False, with_rows: bool = True, st=None,
|
| 2124 |
*, fields_only: bool = False, needed=None, cursor=None, limit=None):
|
|
|
|
|
|
|
|
|
|
| 2125 |
from core import grid_events
|
| 2126 |
actor = grid_events.request_admission(session, table_key + '_table_workspace')
|
| 2127 |
result = _ut_assembly(actor, table_key, storage_key, consume_corrections, with_rows, st,
|
|
|
|
| 3347 |
(found by this route's own gate check, not by luck)."""
|
| 3348 |
import aios_grid
|
| 3349 |
|
|
|
|
|
|
|
| 3350 |
if table_key == 'ut_activity':
|
| 3351 |
if related:
|
| 3352 |
raise err(409, 'activity_related_unavailable', 'Open the Activity database to read events.')
|
|
|
|
| 4521 |
from core import grid_events
|
| 4522 |
|
| 4523 |
_managed_or_refuse(session, table_key, mutation=True)
|
| 4524 |
+
_records_or_refuse(session, table_key)
|
|
|
|
| 4525 |
body = body if isinstance(body, dict) else {}
|
| 4526 |
row_id = str(body.get("rowId") or "").strip()
|
| 4527 |
if not row_id:
|
|
|
|
| 4551 |
raise err(404, "history_event_not_found", "that retained value version is unavailable")
|
| 4552 |
if any(key in (g.get("hidden") or ()) for key in updates):
|
| 4553 |
raise err(403, "field_hidden", "that retained version includes a hidden field")
|
|
|
|
| 4554 |
prior_history_ids = {
|
| 4555 |
str(item.get("id")) for item in _ut().value_history(
|
| 4556 |
table_key, row_id, st=session.runtime, retention_days=days,
|
api/sync_runtime.py
CHANGED
|
@@ -6,7 +6,7 @@ publish is 878.8 s inside the 1,200 s limit, so a pass fits one job.
|
|
| 6 |
Decision (lead ruling, 2026-11-03): the default job is RESIDENT (`sync_worker.py --job`). It runs
|
| 7 |
every worker loop as the old child did, because about ten heavy loops (automation, Amazon ads,
|
| 8 |
inbox, payouts, aged stock, Sheet refresh, restock, returns, materialization, first-load floor)
|
| 9 |
-
live only in the worker. A resident job hands over to a new job only at
|
| 10 |
resync loop, when the web's deployed version differs from its own or under 3 h of its 24 h
|
| 11 |
timeout remain. AIOS_SYNC_JOB_MODE=per-pass keeps one job per pass (`sync_worker.py --once`).
|
| 12 |
The job runs the Space's own image (hf.co/spaces/<SPACE_ID>), proved by HF job
|
|
@@ -17,7 +17,7 @@ AIOS_SYNC_RUNTIME picks the runtime: `hf-job`, `external` (a separately run resi
|
|
| 17 |
HF sets SPACE_HOST in a Space. The DigitalOcean staging config sets SPACE_ID only.
|
| 18 |
|
| 19 |
Web side: `tick` submits a job for each tenant with no live job: always when resident, only when
|
| 20 |
-
its work is due when per-pass. Worker side: `
|
| 21 |
The tenant's C-status `sync_worker` entry carries `job` {id, submittedAt, image, tenant},
|
| 22 |
`schedule` {nextPassAt, at, reason} and `web` {version, at}. Stamps are ISO-8601 UTC, the format
|
| 23 |
of RESYNC_STATE's.
|
|
@@ -29,7 +29,6 @@ import math
|
|
| 29 |
import os
|
| 30 |
import threading
|
| 31 |
import time
|
| 32 |
-
from copy import deepcopy
|
| 33 |
|
| 34 |
RUNTIMES = ("hf-job", "external", "off")
|
| 35 |
RUNTIME_ENV = "AIOS_SYNC_RUNTIME"
|
|
@@ -234,51 +233,8 @@ def relay_due(now=None):
|
|
| 234 |
return ""
|
| 235 |
|
| 236 |
|
| 237 |
-
def _handoff_queued_requests():
|
| 238 |
-
"""Persist ordinary queued receipts without changing durable events or interrupting a claim."""
|
| 239 |
-
from core import connector_sync
|
| 240 |
-
import process_role
|
| 241 |
-
with connector_sync._SYNC_REQUEST_LOCK:
|
| 242 |
-
receipts = deepcopy(connector_sync._SYNC_REQUESTS)
|
| 243 |
-
runtimes = dict(connector_sync._SYNC_REQUEST_RUNTIMES)
|
| 244 |
-
if any(row.get("state") == "running" for row in receipts.values()):
|
| 245 |
-
return None
|
| 246 |
-
try:
|
| 247 |
-
for row in receipts.values():
|
| 248 |
-
if row.get("state") != "queued" or row.get("tables") is not None:
|
| 249 |
-
continue
|
| 250 |
-
tenant = row["tenant"]
|
| 251 |
-
rt = runtimes.get(tenant) or process_role._status_runtime(tenant)
|
| 252 |
-
if rt.key != tenant:
|
| 253 |
-
raise ValueError("The relay desk does not match the receipt tenant.")
|
| 254 |
-
connector_sync.queue_remote_request(rt, row, row.get("connectionId"))
|
| 255 |
-
except Exception as exc:
|
| 256 |
-
print(f"[sync-worker] job: relay handoff failed ({type(exc).__name__}); keeping this job running", flush=True)
|
| 257 |
-
return None
|
| 258 |
-
return receipts
|
| 259 |
-
|
| 260 |
-
|
| 261 |
-
def relay_at_boundary(next_at=None, *, exit_fn=None):
|
| 262 |
-
"""Relay a resident job after its current write and projection finish, before another claim."""
|
| 263 |
-
from core import connector_sync
|
| 264 |
-
if not ONCE["on"] or ONCE["mode"] != "resident":
|
| 265 |
-
return False
|
| 266 |
-
reason = relay_due()
|
| 267 |
-
if not reason:
|
| 268 |
-
return False
|
| 269 |
-
receipts = _handoff_queued_requests()
|
| 270 |
-
if receipts is None:
|
| 271 |
-
return False
|
| 272 |
-
with connector_sync._SYNC_REQUEST_LOCK:
|
| 273 |
-
# No receipt can be admitted or claimed between the final handoff check and exit.
|
| 274 |
-
if connector_sync._SYNC_REQUESTS != receipts:
|
| 275 |
-
return False
|
| 276 |
-
finish(time.time() if next_at is None else next_at, reason, exit_fn=exit_fn)
|
| 277 |
-
return True
|
| 278 |
-
|
| 279 |
-
|
| 280 |
def idle_wait(event, seconds, *, exit_fn=None):
|
| 281 |
-
"""
|
| 282 |
if not ONCE["on"]:
|
| 283 |
return event.wait(seconds)
|
| 284 |
if seconds is None or not math.isfinite(float(seconds)):
|
|
@@ -286,21 +242,18 @@ def idle_wait(event, seconds, *, exit_fn=None):
|
|
| 286 |
deadline = time.time() + max(0.0, min(float(seconds), MAX_IDLE_SECONDS))
|
| 287 |
resident = ONCE["mode"] == "resident"
|
| 288 |
while True:
|
| 289 |
-
if resident and relay_at_boundary(deadline, exit_fn=exit_fn):
|
| 290 |
-
return False
|
| 291 |
remaining = deadline - time.time()
|
| 292 |
if remaining <= IDLE_GRACE_SECONDS:
|
| 293 |
return event.wait(max(0.0, remaining))
|
| 294 |
# Per-pass: the grace lets the desk poll admit a request queued while the pass ran.
|
| 295 |
if event.wait(min(RELAY_CHECK_SECONDS if resident else IDLE_GRACE_SECONDS, remaining)):
|
| 296 |
-
if resident and relay_at_boundary(deadline, exit_fn=exit_fn):
|
| 297 |
-
return False
|
| 298 |
return True
|
| 299 |
-
if resident
|
|
|
|
| 300 |
continue
|
| 301 |
busy, wake_at = _desk_state(time.time())
|
| 302 |
if not busy:
|
| 303 |
-
finish(deadline if wake_at is None else min(deadline, wake_at),
|
| 304 |
return False
|
| 305 |
|
| 306 |
|
|
@@ -345,7 +298,7 @@ def no_lease_line(tenant, seconds=None):
|
|
| 345 |
|
| 346 |
|
| 347 |
def _pass_in_flight():
|
| 348 |
-
"""True while the store resync loop runs
|
| 349 |
return any(t.name == "store-resync" and t.is_alive() for t in threading.enumerate())
|
| 350 |
|
| 351 |
|
|
@@ -356,7 +309,9 @@ def run_once(*, exit_fn=None, alive=None):
|
|
| 356 |
if alive():
|
| 357 |
continue
|
| 358 |
if ONCE["mode"] == "resident":
|
| 359 |
-
|
|
|
|
|
|
|
| 360 |
return False
|
| 361 |
continue
|
| 362 |
finish(time.time() + MAX_IDLE_SECONDS,
|
|
|
|
| 6 |
Decision (lead ruling, 2026-11-03): the default job is RESIDENT (`sync_worker.py --job`). It runs
|
| 7 |
every worker loop as the old child did, because about ten heavy loops (automation, Amazon ads,
|
| 8 |
inbox, payouts, aged stock, Sheet refresh, restock, returns, materialization, first-load floor)
|
| 9 |
+
live only in the worker. A resident job hands over to a new job only at an idle point of the
|
| 10 |
resync loop, when the web's deployed version differs from its own or under 3 h of its 24 h
|
| 11 |
timeout remain. AIOS_SYNC_JOB_MODE=per-pass keeps one job per pass (`sync_worker.py --once`).
|
| 12 |
The job runs the Space's own image (hf.co/spaces/<SPACE_ID>), proved by HF job
|
|
|
|
| 17 |
HF sets SPACE_HOST in a Space. The DigitalOcean staging config sets SPACE_ID only.
|
| 18 |
|
| 19 |
Web side: `tick` submits a job for each tenant with no live job: always when resident, only when
|
| 20 |
+
its work is due when per-pass. Worker side: `idle_wait` is the resync loop's only exit point.
|
| 21 |
The tenant's C-status `sync_worker` entry carries `job` {id, submittedAt, image, tenant},
|
| 22 |
`schedule` {nextPassAt, at, reason} and `web` {version, at}. Stamps are ISO-8601 UTC, the format
|
| 23 |
of RESYNC_STATE's.
|
|
|
|
| 29 |
import os
|
| 30 |
import threading
|
| 31 |
import time
|
|
|
|
| 32 |
|
| 33 |
RUNTIMES = ("hf-job", "external", "off")
|
| 34 |
RUNTIME_ENV = "AIOS_SYNC_RUNTIME"
|
|
|
|
| 233 |
return ""
|
| 234 |
|
| 235 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 236 |
def idle_wait(event, seconds, *, exit_fn=None):
|
| 237 |
+
"""The resync loop's wait, and a job's only exit point. Outside a job it is `event.wait(seconds)`."""
|
| 238 |
if not ONCE["on"]:
|
| 239 |
return event.wait(seconds)
|
| 240 |
if seconds is None or not math.isfinite(float(seconds)):
|
|
|
|
| 242 |
deadline = time.time() + max(0.0, min(float(seconds), MAX_IDLE_SECONDS))
|
| 243 |
resident = ONCE["mode"] == "resident"
|
| 244 |
while True:
|
|
|
|
|
|
|
| 245 |
remaining = deadline - time.time()
|
| 246 |
if remaining <= IDLE_GRACE_SECONDS:
|
| 247 |
return event.wait(max(0.0, remaining))
|
| 248 |
# Per-pass: the grace lets the desk poll admit a request queued while the pass ran.
|
| 249 |
if event.wait(min(RELAY_CHECK_SECONDS if resident else IDLE_GRACE_SECONDS, remaining)):
|
|
|
|
|
|
|
| 250 |
return True
|
| 251 |
+
reason = relay_due() if resident else "idle"
|
| 252 |
+
if not reason:
|
| 253 |
continue
|
| 254 |
busy, wake_at = _desk_state(time.time())
|
| 255 |
if not busy:
|
| 256 |
+
finish(deadline if wake_at is None else min(deadline, wake_at), reason, exit_fn=exit_fn)
|
| 257 |
return False
|
| 258 |
|
| 259 |
|
|
|
|
| 298 |
|
| 299 |
|
| 300 |
def _pass_in_flight():
|
| 301 |
+
"""True while the store resync loop runs: it relays at its own idle point."""
|
| 302 |
return any(t.name == "store-resync" and t.is_alive() for t in threading.enumerate())
|
| 303 |
|
| 304 |
|
|
|
|
| 309 |
if alive():
|
| 310 |
continue
|
| 311 |
if ONCE["mode"] == "resident":
|
| 312 |
+
reason = relay_due()
|
| 313 |
+
if reason and not _desk_state(time.time())[0]:
|
| 314 |
+
finish(time.time(), reason, exit_fn=exit_fn)
|
| 315 |
return False
|
| 316 |
continue
|
| 317 |
finish(time.time() + MAX_IDLE_SECONDS,
|
api/test_odoo_rank_refresh.py
CHANGED
|
@@ -1059,7 +1059,7 @@ class RankRefreshTests(unittest.TestCase):
|
|
| 1059 |
|
| 1060 |
def run_slot_loop(self, memory, *, boot, max_sleeps, tenant='royal-imports', config=None,
|
| 1061 |
reports=None, checkpoints=None, take=None, slot_flags=None, status=None,
|
| 1062 |
-
stop_at=None):
|
| 1063 |
"""Drive the real loop and its real slot resolver on a clock that advances at each wait."""
|
| 1064 |
import odoo_relational as rel
|
| 1065 |
source = ast.parse(Path(__file__).with_name('main.py').read_text(encoding='utf-8'))
|
|
@@ -1086,6 +1086,8 @@ class RankRefreshTests(unittest.TestCase):
|
|
| 1086 |
|
| 1087 |
def sync(manual_flag, passes, entities):
|
| 1088 |
events.append(('mirror', now[0], manual_flag))
|
|
|
|
|
|
|
| 1089 |
if manual_flag:
|
| 1090 |
return None, None, None
|
| 1091 |
if reports:
|
|
@@ -1684,5 +1686,72 @@ class RankRefreshTests(unittest.TestCase):
|
|
| 1684 |
self.assertEqual((memory['slotResumedFor'], memory['slotResumes']), (slot_at, 1))
|
| 1685 |
|
| 1686 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1687 |
if __name__ == '__main__':
|
| 1688 |
unittest.main()
|
|
|
|
| 1059 |
|
| 1060 |
def run_slot_loop(self, memory, *, boot, max_sleeps, tenant='royal-imports', config=None,
|
| 1061 |
reports=None, checkpoints=None, take=None, slot_flags=None, status=None,
|
| 1062 |
+
stop_at=None, on_pass=None):
|
| 1063 |
"""Drive the real loop and its real slot resolver on a clock that advances at each wait."""
|
| 1064 |
import odoo_relational as rel
|
| 1065 |
source = ast.parse(Path(__file__).with_name('main.py').read_text(encoding='utf-8'))
|
|
|
|
| 1086 |
|
| 1087 |
def sync(manual_flag, passes, entities):
|
| 1088 |
events.append(('mirror', now[0], manual_flag))
|
| 1089 |
+
if on_pass is not None:
|
| 1090 |
+
on_pass(store, now[0])
|
| 1091 |
if manual_flag:
|
| 1092 |
return None, None, None
|
| 1093 |
if reports:
|
|
|
|
| 1686 |
self.assertEqual((memory['slotResumedFor'], memory['slotResumes']), (slot_at, 1))
|
| 1687 |
|
| 1688 |
|
| 1689 |
+
def test_h1_freshness_reads_the_keychain_default_slot(self):
|
| 1690 |
+
"""H1: a pass written under the keychain default id reads fresh; no id keeps the default slot"""
|
| 1691 |
+
from datetime import datetime, timezone
|
| 1692 |
+
import odoo_relational as rel
|
| 1693 |
+
from core import connector_sync
|
| 1694 |
+
now = datetime(2026, 11, 18, 9, 0, tzinfo=timezone.utc)
|
| 1695 |
+
good = '2026-11-18T07:32:38+00:00'
|
| 1696 |
+
entry = {'lastGoodSyncAt': good, 'resync': {'connectionId': 'k_x'},
|
| 1697 |
+
'pass': {'startedAt': '2026-11-18T07:00:00+00:00', 'endedAt': good,
|
| 1698 |
+
'publishedAt': good, 'outcome': 'held'}}
|
| 1699 |
+
keyed = self.SlotStore('acme', {connector_sync.STATUS_KEY: {'odoo': {'k_x': dict(entry)}}})
|
| 1700 |
+
with patch.object(rel, 'default_connection_id', return_value='k_x'):
|
| 1701 |
+
fresh = rel.sync_freshness(keyed, now=now)
|
| 1702 |
+
self.assertFalse(fresh['stale'])
|
| 1703 |
+
self.assertEqual(fresh['lastGoodSyncAt'], good)
|
| 1704 |
+
self.assertEqual((fresh['pass']['outcome'], fresh['resync']), ('held', {'connectionId': 'k_x'}))
|
| 1705 |
+
# Control: read through the "default" slot alone, the same store is stale (the live defect).
|
| 1706 |
+
with patch.object(rel, 'default_connection_id', return_value=None):
|
| 1707 |
+
self.assertTrue(rel.sync_freshness(keyed, now=now)['stale'])
|
| 1708 |
+
# Negative control: an empty keychain slot, or no keychain id, still reads the "default" slot.
|
| 1709 |
+
plain = self.SlotStore('acme', {connector_sync.STATUS_KEY: {'odoo': {'default': dict(entry)}}})
|
| 1710 |
+
for cid in (None, 'k_x'):
|
| 1711 |
+
with patch.object(rel, 'default_connection_id', return_value=cid):
|
| 1712 |
+
fresh = rel.sync_freshness(plain, now=now)
|
| 1713 |
+
self.assertFalse(fresh['stale'], cid)
|
| 1714 |
+
self.assertEqual(fresh['lastGoodSyncAt'], good, cid)
|
| 1715 |
+
|
| 1716 |
+
def test_h1_a_held_pass_that_published_records_its_success(self):
|
| 1717 |
+
"""H1: a held pass that published stamps lastSuccessAt and the source stage; a refused one writes nothing"""
|
| 1718 |
+
import odoo_relational as rel
|
| 1719 |
+
import routes_connectors
|
| 1720 |
+
from core import connector_sync
|
| 1721 |
+
last = self.utc('2026-10-22T06:05:00Z')
|
| 1722 |
+
boot = self.utc('2026-10-24T07:00:00Z')
|
| 1723 |
+
memory = {'passAt': last, 'fullRebuildAt': last, 'fingerprints': {'res_partner': (3, 7)}}
|
| 1724 |
+
held = {'account_move_line': {'phase': 'backfill', 'pulled': 0}}
|
| 1725 |
+
marker = ('account_move_line.cols.amount_residual', 'running', 500, None, 100_000)
|
| 1726 |
+
|
| 1727 |
+
def publication(offset):
|
| 1728 |
+
def write(store, at):
|
| 1729 |
+
row = {'startedAt': connector_sync._utc_stamp(at), 'outcome': 'running'}
|
| 1730 |
+
if offset is not None:
|
| 1731 |
+
row.update(outcome='done', publishedAt=connector_sync._utc_stamp(at + offset))
|
| 1732 |
+
store.docs[connector_sync.STATUS_KEY] = {'odoo': {'k_x': {'pass': row}}}
|
| 1733 |
+
return write
|
| 1734 |
+
# Published during this pass; published by an earlier pass; publication refused.
|
| 1735 |
+
for offset, published in ((5, True), (-3600, False), (None, False)):
|
| 1736 |
+
stage = SimpleNamespace(calls=[])
|
| 1737 |
+
with patch.object(rel, 'default_connection_id', return_value='k_x'), \
|
| 1738 |
+
patch.object(routes_connectors, 'record_odoo_stage',
|
| 1739 |
+
lambda *a, **k: stage.calls.append((a, k))):
|
| 1740 |
+
self.run_slot_loop(memory, boot=boot, max_sleeps=1, reports=[held],
|
| 1741 |
+
checkpoints=[(marker,), (marker,)], on_pass=publication(offset))
|
| 1742 |
+
self.assertTrue(self.slot_state['publicationHeld'], offset)
|
| 1743 |
+
self.assertEqual(self.slot_state['connectionId'], 'k_x')
|
| 1744 |
+
self.assertIn('publication held; no vendor pulls or relational rebuild', self.loop_output)
|
| 1745 |
+
if published:
|
| 1746 |
+
self.assertEqual(self.slot_state.get('lastSuccessAt'), self.slot_state['lastPassAt'])
|
| 1747 |
+
self.assertEqual(len(stage.calls), 1)
|
| 1748 |
+
args, kwargs = stage.calls[0]
|
| 1749 |
+
self.assertEqual(args[1:], ('k_x', 'source', 'ready'))
|
| 1750 |
+
self.assertEqual(kwargs['at'], connector_sync._utc_stamp(boot + 5))
|
| 1751 |
+
else:
|
| 1752 |
+
self.assertNotIn('lastSuccessAt', self.slot_state, offset)
|
| 1753 |
+
self.assertEqual(stage.calls, [], offset)
|
| 1754 |
+
|
| 1755 |
+
|
| 1756 |
if __name__ == '__main__':
|
| 1757 |
unittest.main()
|
api/usage_ledger.py
CHANGED
|
@@ -3,13 +3,16 @@
|
|
| 3 |
R9: *"ONE meter for every AI surface. The dashboard REPORTS weekly usage against an allowance and
|
| 4 |
does NOT cut anyone off this wave; over the allowance it still works, shown red."*
|
| 5 |
|
| 6 |
-
⛔ THIS FILE MEASURES AND NEVER ENFORCES.
|
| 7 |
-
|
|
|
|
|
|
|
| 8 |
|
| 9 |
-
**THE SURFACES.** `assistant` (a person asking the Assistant a question) · `field_agent` (an AI
|
| 10 |
enrichment column filling cells) · `ai_review` (a model deciding a review stage) · `automation_draft`
|
| 11 |
-
(a sentence becoming a draft flow)
|
| 12 |
-
|
|
|
|
| 13 |
|
| 14 |
⛔⛔ **`record()` MUST BE HANDED ITS TARGET, AND THAT IS A MEASUREMENT, NOT A STYLE CHOICE.** The
|
| 15 |
obvious design is to bind `(runtime, username)` into a `contextvars.ContextVar` inside
|
|
@@ -29,17 +32,17 @@ that hides the gap is worse than one that admits it.
|
|
| 29 |
"""
|
| 30 |
from __future__ import annotations
|
| 31 |
|
| 32 |
-
from decimal import Decimal, InvalidOperation, ROUND_HALF_UP
|
| 33 |
import os
|
| 34 |
import time
|
| 35 |
|
| 36 |
-
#:
|
|
|
|
| 37 |
STORE_KEY = "ai_usage"
|
| 38 |
|
| 39 |
-
#: The AI surfaces (C7). ⚠ A surface this list does not name is REFUSED rather than stored under
|
| 40 |
#: whatever string arrived, because a typo'd surface is a meter that reads low forever with a second
|
| 41 |
#: row nobody looks at.
|
| 42 |
-
SURFACES = ("assistant", "field_agent", "ai_review", "automation_draft"
|
| 43 |
|
| 44 |
#: The weekly allowance the dashboard measures against (R9). ⛔ IT IS A REPORTING LINE, NOT A GATE.
|
| 45 |
#: Env-tunable so a deployment can set a real number without a release.
|
|
@@ -129,39 +132,26 @@ def _int_or_none(value):
|
|
| 129 |
return None
|
| 130 |
|
| 131 |
|
| 132 |
-
def usd_micros(value):
|
| 133 |
-
"""Return nonnegative dollars as whole micro-dollars, rounded half up, or None if unknown."""
|
| 134 |
-
if isinstance(value, bool) or not isinstance(value, (int, float, str, Decimal)):
|
| 135 |
-
return None
|
| 136 |
-
try:
|
| 137 |
-
amount = Decimal(str(value))
|
| 138 |
-
if amount.is_finite() and amount >= 0:
|
| 139 |
-
return int((amount * 1_000_000).to_integral_value(rounding=ROUND_HALF_UP))
|
| 140 |
-
except (InvalidOperation, ValueError, OverflowError):
|
| 141 |
-
return None
|
| 142 |
-
return None
|
| 143 |
-
|
| 144 |
-
|
| 145 |
-
def _book_dollars(row, micros, calls, entry_id):
|
| 146 |
-
"""Book measured dollars or a separate unknown count, resolving a single-call receipt once."""
|
| 147 |
-
receipts = row.setdefault("_cost_entries", {}) if entry_id else {}
|
| 148 |
-
repeated = bool(entry_id and entry_id in receipts)
|
| 149 |
-
if repeated and (receipts[entry_id] is not None or micros is None):
|
| 150 |
-
return False
|
| 151 |
-
if repeated:
|
| 152 |
-
row["usd_unmeasured"] = max(0, int(row.get("usd_unmeasured") or 0) - 1)
|
| 153 |
-
if micros is None:
|
| 154 |
-
row["usd_unmeasured"] = int(row.get("usd_unmeasured") or 0) + calls
|
| 155 |
-
else:
|
| 156 |
-
row["usd_micros"] = int(row.get("usd_micros") or 0) + micros
|
| 157 |
-
if entry_id:
|
| 158 |
-
receipts[entry_id] = micros
|
| 159 |
-
return not repeated
|
| 160 |
-
|
| 161 |
-
|
| 162 |
def record(surface, provider, model, tokens_in=None, tokens_out=None, *,
|
| 163 |
-
total=None, calls=1,
|
| 164 |
-
"""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 165 |
key = str(surface or "").strip()
|
| 166 |
if key not in SURFACES:
|
| 167 |
# Refused rather than stored: a surface nobody named is a row nobody reads, and the meter
|
|
@@ -172,16 +162,15 @@ def record(surface, provider, model, tokens_in=None, tokens_out=None, *,
|
|
| 172 |
if tot is None and (ins is not None or outs is not None):
|
| 173 |
tot = (ins or 0) + (outs or 0)
|
| 174 |
n = max(0, int(calls) if isinstance(calls, int) and not isinstance(calls, bool) else 1)
|
| 175 |
-
if not n
|
| 176 |
return False
|
| 177 |
uname = str(user or "").strip().lower()
|
| 178 |
if st is None:
|
| 179 |
UNATTRIBUTED["calls"] += n
|
| 180 |
UNATTRIBUTED["surfaces"][key] = UNATTRIBUTED["surfaces"].get(key, 0) + n
|
| 181 |
return False
|
| 182 |
-
week =
|
| 183 |
prov, mdl = str(provider or "")[:40], str(model or "")[:80]
|
| 184 |
-
micros = usd_micros(cost_usd)
|
| 185 |
|
| 186 |
def _up(data):
|
| 187 |
data = data if isinstance(data, dict) else {}
|
|
@@ -189,16 +178,18 @@ def record(surface, provider, model, tokens_in=None, tokens_out=None, *,
|
|
| 189 |
row = ((weeks.setdefault(week, {})
|
| 190 |
.setdefault(uname or "-", {}))
|
| 191 |
.setdefault(key, {}))
|
| 192 |
-
|
| 193 |
-
|
| 194 |
-
|
| 195 |
-
|
| 196 |
-
|
| 197 |
-
|
| 198 |
-
|
| 199 |
-
|
| 200 |
-
|
| 201 |
-
|
|
|
|
|
|
|
| 202 |
if prov:
|
| 203 |
row["provider"] = prov
|
| 204 |
if mdl:
|
|
@@ -210,8 +201,12 @@ def record(surface, provider, model, tokens_in=None, tokens_out=None, *,
|
|
| 210 |
return weeks
|
| 211 |
|
| 212 |
try:
|
| 213 |
-
#
|
| 214 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 215 |
return True
|
| 216 |
except Exception: # noqa: BLE001
|
| 217 |
UNATTRIBUTED["calls"] += n
|
|
@@ -220,8 +215,7 @@ def record(surface, provider, model, tokens_in=None, tokens_out=None, *,
|
|
| 220 |
|
| 221 |
|
| 222 |
def _blank_row():
|
| 223 |
-
return {"calls": 0, "tokens": 0, "tokens_in": 0, "tokens_out": 0, "unmeasured": 0
|
| 224 |
-
"usd_micros": 0, "usd_unmeasured": 0}
|
| 225 |
|
| 226 |
|
| 227 |
def _fold(rows):
|
|
@@ -229,7 +223,7 @@ def _fold(rows):
|
|
| 229 |
for row in rows:
|
| 230 |
if not isinstance(row, dict):
|
| 231 |
continue
|
| 232 |
-
for field in
|
| 233 |
got = row.get(field)
|
| 234 |
if isinstance(got, int) and not isinstance(got, bool):
|
| 235 |
out[field] += max(0, got)
|
|
|
|
| 3 |
R9: *"ONE meter for every AI surface. The dashboard REPORTS weekly usage against an allowance and
|
| 4 |
does NOT cut anyone off this wave; over the allowance it still works, shown red."*
|
| 5 |
|
| 6 |
+
⛔ SO THIS FILE MEASURES AND NEVER ENFORCES. There is no ceiling here, no refusal, and no caller of
|
| 7 |
+
this module may branch on its answer. The one limit the product does enforce is per COLUMN and lives
|
| 8 |
+
where it was built (`ai_enrich.ceiling_report`); this is the tenant-wide REPORT, and mixing the two
|
| 9 |
+
would make an over-allowance week silently stop somebody's enrichment run.
|
| 10 |
|
| 11 |
+
**THE FOUR SURFACES.** `assistant` (a person asking the Assistant a question) · `field_agent` (an AI
|
| 12 |
enrichment column filling cells) · `ai_review` (a model deciding a review stage) · `automation_draft`
|
| 13 |
+
(a sentence becoming a draft flow). They are the four LLM entry points the product has, and the gate
|
| 14 |
+
in `verify_api.py` DERIVES them from the code rather than reading this list, so a fifth entry point
|
| 15 |
+
is caught by the gate and not by somebody remembering to edit a tuple.
|
| 16 |
|
| 17 |
⛔⛔ **`record()` MUST BE HANDED ITS TARGET, AND THAT IS A MEASUREMENT, NOT A STYLE CHOICE.** The
|
| 18 |
obvious design is to bind `(runtime, username)` into a `contextvars.ContextVar` inside
|
|
|
|
| 32 |
"""
|
| 33 |
from __future__ import annotations
|
| 34 |
|
|
|
|
| 35 |
import os
|
| 36 |
import time
|
| 37 |
|
| 38 |
+
#: The store bucket. Small by construction — counters only, never a per-call log: a log would grow
|
| 39 |
+
#: without bound on a route that runs per row, and nothing in R9 asks a question a log answers.
|
| 40 |
STORE_KEY = "ai_usage"
|
| 41 |
|
| 42 |
+
#: The four AI surfaces (C7). ⚠ A surface this list does not name is REFUSED rather than stored under
|
| 43 |
#: whatever string arrived, because a typo'd surface is a meter that reads low forever with a second
|
| 44 |
#: row nobody looks at.
|
| 45 |
+
SURFACES = ("assistant", "field_agent", "ai_review", "automation_draft")
|
| 46 |
|
| 47 |
#: The weekly allowance the dashboard measures against (R9). ⛔ IT IS A REPORTING LINE, NOT A GATE.
|
| 48 |
#: Env-tunable so a deployment can set a real number without a release.
|
|
|
|
| 132 |
return None
|
| 133 |
|
| 134 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 135 |
def record(surface, provider, model, tokens_in=None, tokens_out=None, *,
|
| 136 |
+
total=None, calls=1, st=None, user=""):
|
| 137 |
+
"""Book ONE AI call. Returns True when the line landed durably, False when it could not.
|
| 138 |
+
|
| 139 |
+
C7's signature is the five positional parameters; everything after `*` is this module's own
|
| 140 |
+
extension and each one has a reason:
|
| 141 |
+
|
| 142 |
+
· `total` — for a caller that knows the total and not the split (`ai_enrich.run_field`
|
| 143 |
+
accumulates a per-run total through an `_ask` contract a gate injects against, and changing
|
| 144 |
+
that contract to carry a split would break the injected shape for no gain here).
|
| 145 |
+
· `calls` — so a RUN over many rows books one line with its real call count instead of one
|
| 146 |
+
store write per row.
|
| 147 |
+
· `st` / `user` — the target. Required in practice; see this module's header for the measured
|
| 148 |
+
reason a contextvar cannot supply them.
|
| 149 |
+
|
| 150 |
+
⛔ IT NEVER RAISES. A meter that can break the feature it measures is worse than no meter: a
|
| 151 |
+
store blip during an enrichment run would abort the run and lose the cells.
|
| 152 |
+
⛔ AND IT NEVER SILENTLY SUCCEEDS. A call it cannot attribute increments `UNATTRIBUTED`, which
|
| 153 |
+
`GET /usage` shows an admin.
|
| 154 |
+
"""
|
| 155 |
key = str(surface or "").strip()
|
| 156 |
if key not in SURFACES:
|
| 157 |
# Refused rather than stored: a surface nobody named is a row nobody reads, and the meter
|
|
|
|
| 162 |
if tot is None and (ins is not None or outs is not None):
|
| 163 |
tot = (ins or 0) + (outs or 0)
|
| 164 |
n = max(0, int(calls) if isinstance(calls, int) and not isinstance(calls, bool) else 1)
|
| 165 |
+
if not n:
|
| 166 |
return False
|
| 167 |
uname = str(user or "").strip().lower()
|
| 168 |
if st is None:
|
| 169 |
UNATTRIBUTED["calls"] += n
|
| 170 |
UNATTRIBUTED["surfaces"][key] = UNATTRIBUTED["surfaces"].get(key, 0) + n
|
| 171 |
return False
|
| 172 |
+
week = week_of()
|
| 173 |
prov, mdl = str(provider or "")[:40], str(model or "")[:80]
|
|
|
|
| 174 |
|
| 175 |
def _up(data):
|
| 176 |
data = data if isinstance(data, dict) else {}
|
|
|
|
| 178 |
row = ((weeks.setdefault(week, {})
|
| 179 |
.setdefault(uname or "-", {}))
|
| 180 |
.setdefault(key, {}))
|
| 181 |
+
row["calls"] = int(row.get("calls") or 0) + n
|
| 182 |
+
if tot is None:
|
| 183 |
+
# ⛔ "the call happened, the tokens are unknown" IS A STATE, and it has to be
|
| 184 |
+
# representable or the meter quietly under-reports. The dashboard shows it as a count of
|
| 185 |
+
# unmeasured calls beside the token total.
|
| 186 |
+
row["unmeasured"] = int(row.get("unmeasured") or 0) + n
|
| 187 |
+
else:
|
| 188 |
+
row["tokens"] = int(row.get("tokens") or 0) + tot
|
| 189 |
+
if ins is not None:
|
| 190 |
+
row["tokens_in"] = int(row.get("tokens_in") or 0) + ins
|
| 191 |
+
if outs is not None:
|
| 192 |
+
row["tokens_out"] = int(row.get("tokens_out") or 0) + outs
|
| 193 |
if prov:
|
| 194 |
row["provider"] = prov
|
| 195 |
if mdl:
|
|
|
|
| 201 |
return weeks
|
| 202 |
|
| 203 |
try:
|
| 204 |
+
# ⚠ `flush='async'` — the coalescing mode ([[store-async-flush]]). This is the hot path of
|
| 205 |
+
# every AI surface in the product (an enrichment run books one line per RUN, but the
|
| 206 |
+
# assistant books one per question), and a blocking hub upload per call would put a network
|
| 207 |
+
# round trip inside the answer a person is waiting for. The mutation applies to the
|
| 208 |
+
# in-process cache immediately, so `GET /usage` reads its own writes.
|
| 209 |
+
st.update(STORE_KEY, _up, flush="async")
|
| 210 |
return True
|
| 211 |
except Exception: # noqa: BLE001
|
| 212 |
UNATTRIBUTED["calls"] += n
|
|
|
|
| 215 |
|
| 216 |
|
| 217 |
def _blank_row():
|
| 218 |
+
return {"calls": 0, "tokens": 0, "tokens_in": 0, "tokens_out": 0, "unmeasured": 0}
|
|
|
|
| 219 |
|
| 220 |
|
| 221 |
def _fold(rows):
|
|
|
|
| 223 |
for row in rows:
|
| 224 |
if not isinstance(row, dict):
|
| 225 |
continue
|
| 226 |
+
for field in ("calls", "tokens", "tokens_in", "tokens_out", "unmeasured"):
|
| 227 |
got = row.get(field)
|
| 228 |
if isinstance(got, int) and not isinstance(got, bool):
|
| 229 |
out[field] += max(0, got)
|
platform/core/amazon_projects.py
CHANGED
|
@@ -26,7 +26,6 @@ STATUSES = ('Backlog', 'Planned', 'In progress', 'Completed', 'Canceled')
|
|
| 26 |
COMPLETED = 'Completed'
|
| 27 |
CANCELED = 'Canceled'
|
| 28 |
PRIORITIES = ('No priority', 'Urgent', 'High', 'Medium', 'Low')
|
| 29 |
-
TYPES = ('project', 'pdp')
|
| 30 |
#: Health turns At risk this many days before the target date.
|
| 31 |
AT_RISK_DAYS = 7
|
| 32 |
CHUNK = 200
|
|
@@ -244,8 +243,6 @@ def _record(values, columns=PROJECT_COLUMNS):
|
|
| 244 |
value = values[len(columns) + 3] if len(values) > len(columns) + 3 else None
|
| 245 |
members = json.loads(value) if isinstance(value, str) else value
|
| 246 |
result['contributors'] = sorted(members or [], key=lambda member: member['username'])
|
| 247 |
-
result['type'] = values[len(columns) + 4] if len(values) > len(columns) + 4 else 'project'
|
| 248 |
-
result['listed'] = bool(values[len(columns) + 5]) if len(values) > len(columns) + 5 else True
|
| 249 |
return result
|
| 250 |
|
| 251 |
|
|
@@ -751,53 +748,11 @@ def _activity_target_preview(value, truncated):
|
|
| 751 |
return f'{shown}. Target list incomplete' if shown else 'Target list incomplete'
|
| 752 |
|
| 753 |
|
| 754 |
-
def _pdp_item_stages(value, pdp):
|
| 755 |
-
"""Accept complete typed workflow item identities before comparing historical stages."""
|
| 756 |
-
if not isinstance(value, dict) or value.get('type') != 'pdp' or not isinstance(value.get('items'), list):
|
| 757 |
-
return None
|
| 758 |
-
stages = {}
|
| 759 |
-
for item in value['items']:
|
| 760 |
-
if (not isinstance(item, dict) or not isinstance(item.get('asin'), str)
|
| 761 |
-
or not isinstance(item.get('stage'), str) or item['stage'] not in pdp.STAGE_PROGRESS
|
| 762 |
-
or item['asin'] in stages):
|
| 763 |
-
return None
|
| 764 |
-
stages[item['asin']] = item['stage']
|
| 765 |
-
return stages
|
| 766 |
-
|
| 767 |
-
|
| 768 |
-
def _pdp_activity_facts(event, field, before, after):
|
| 769 |
-
"""Project only C4 action facts retained in the bounded stored event."""
|
| 770 |
-
from core import amazon_pdp as pdp
|
| 771 |
-
if event.get('via') != pdp.VIA:
|
| 772 |
-
return None
|
| 773 |
-
unavailable = {'action': 'unavailable'}
|
| 774 |
-
if event.get('changes_truncated'):
|
| 775 |
-
return unavailable
|
| 776 |
-
old, new = (_pdp_item_stages(value, pdp) for value in (before, after))
|
| 777 |
-
if field == 'setup' and before is None and new is not None:
|
| 778 |
-
scope = after.get('scope')
|
| 779 |
-
scope = scope if isinstance(scope, list) and scope and all(key in pdp.SCOPE_PARTS for key in scope) else None
|
| 780 |
-
return {'action': 'created', 'scope': scope}
|
| 781 |
-
if old is not None and new is not None:
|
| 782 |
-
changed = {stage for asin, stage in new.items() if asin in old and old[asin] != stage}
|
| 783 |
-
if len(changed) == 1:
|
| 784 |
-
stage = next(iter(changed))
|
| 785 |
-
if field == ('publish' if stage == 'published' else stage):
|
| 786 |
-
return {'action': 'moved', 'stage': stage} if stage != 'published' else {'action': 'published'}
|
| 787 |
-
return {'action': 'setup'} if field in ('setup', 'competitors') and not changed else unavailable
|
| 788 |
-
if isinstance(after, dict) and isinstance(after.get('slot'), str) and after['slot'] in pdp.SLOTS:
|
| 789 |
-
part = pdp.SLOTS[after['slot']]['key']
|
| 790 |
-
if after.get('part') == part and field in ('draft', 'approve', 'request'):
|
| 791 |
-
return {'action': {'draft': 'saved', 'approve': 'approved', 'request': 'requested'}[field]}
|
| 792 |
-
return {'action': 'ai'} if field == 'ai' else unavailable
|
| 793 |
-
|
| 794 |
-
|
| 795 |
def _activity_entries(event, actor_names, task_names):
|
| 796 |
"""Project one log event into named activity entries."""
|
| 797 |
out = []
|
| 798 |
for index, (field, change) in enumerate((event.get('changes') or {}).items()):
|
| 799 |
before, after = change.get('before'), change.get('after')
|
| 800 |
-
pdp_facts = _pdp_activity_facts(event, field, before, after)
|
| 801 |
if field == 'project':
|
| 802 |
action, field = ('created' if before is None else 'deleted'), None
|
| 803 |
elif field == 'targets':
|
|
@@ -820,8 +775,6 @@ def _activity_entries(event, actor_names, task_names):
|
|
| 820 |
'by': event['by'], 'byUsername': event['by'],
|
| 821 |
'byName': actor_names.get(event['by'], event['by']),
|
| 822 |
'action': action, 'field': field, 'from': shown[0], 'to': shown[1]})
|
| 823 |
-
if pdp_facts is not None:
|
| 824 |
-
out[-1]['pdp'] = pdp_facts
|
| 825 |
return out
|
| 826 |
|
| 827 |
|
|
@@ -1142,8 +1095,6 @@ class PgProjectStore:
|
|
| 1142 |
notes TEXT, completed_on DATE, source_id TEXT,
|
| 1143 |
created_at TEXT NOT NULL, updated_at TEXT NOT NULL, created_by TEXT,
|
| 1144 |
rollup JSONB, impact JSONB,
|
| 1145 |
-
type TEXT NOT NULL DEFAULT 'project' CHECK (type IN ({_in_sql(TYPES)})),
|
| 1146 |
-
listed BOOLEAN NOT NULL DEFAULT TRUE,
|
| 1147 |
PRIMARY KEY (tenant, id), UNIQUE (tenant, source_id),
|
| 1148 |
CHECK ((status = '{COMPLETED}' AND completed_on IS NOT NULL)
|
| 1149 |
OR (status <> '{COMPLETED}' AND completed_on IS NULL)))''')
|
|
@@ -1161,11 +1112,8 @@ class PgProjectStore:
|
|
| 1161 |
# One row's lookup is one indexed read on this index.
|
| 1162 |
con.execute(f'CREATE INDEX IF NOT EXISTS "amazon_project_targets_entity" '
|
| 1163 |
f'ON {self.t(TARGETS)} (tenant, grain, key, project_id)')
|
| 1164 |
-
|
| 1165 |
-
|
| 1166 |
-
'listed': 'BOOLEAN NOT NULL DEFAULT TRUE'}
|
| 1167 |
-
for field, declaration in additions.items():
|
| 1168 |
-
con.execute(f'ALTER TABLE {self.t(PROJECTS)} ADD COLUMN IF NOT EXISTS {field} {declaration}')
|
| 1169 |
|
| 1170 |
@contextmanager
|
| 1171 |
def _write(self):
|
|
@@ -1178,20 +1126,13 @@ class PgProjectStore:
|
|
| 1178 |
cols = ', '.join(f'p.{c}' for c in PROJECT_COLUMNS)
|
| 1179 |
count = (f'(SELECT count(*) FROM {self.t(TARGETS)} AS c '
|
| 1180 |
'WHERE c.tenant = p.tenant AND c.project_id = p.id)')
|
| 1181 |
-
return (f'SELECT {prefix}{cols}, {count}, {self._rollup_projection()}, {members}
|
| 1182 |
-
f'{self._type_projection()}, {self._listed_projection()} '
|
| 1183 |
f'FROM {self.t(PROJECTS)} AS p WHERE p.tenant = %s{where}')
|
| 1184 |
|
| 1185 |
def _rollup_projection(self):
|
| 1186 |
# Old schemas read blank figures until a guarded write adds the columns.
|
| 1187 |
return "to_jsonb(p)->'rollup', to_jsonb(p)->'impact'"
|
| 1188 |
|
| 1189 |
-
def _type_projection(self):
|
| 1190 |
-
return "COALESCE(to_jsonb(p)->>'type', 'project')"
|
| 1191 |
-
|
| 1192 |
-
def _listed_projection(self):
|
| 1193 |
-
return "COALESCE((to_jsonb(p)->>'listed')::boolean, TRUE)"
|
| 1194 |
-
|
| 1195 |
def _members_projection(self, con):
|
| 1196 |
if con.execute('SELECT to_regclass(%s)', (self.t(MEMBERS),)).fetchone()[0] is None:
|
| 1197 |
return 'NULL'
|
|
@@ -1237,19 +1178,15 @@ class PgProjectStore:
|
|
| 1237 |
'WHERE tenant = %s AND id = %s',
|
| 1238 |
(stamp, json.dumps(impact), self.tenant_slug, project_id))
|
| 1239 |
|
| 1240 |
-
def _insert(self, con, record, pairs, stamp):
|
| 1241 |
-
columns = (*PROJECT_COLUMNS, 'impact', 'type', 'listed')
|
| 1242 |
-
values = (*[record[c] for c in PROJECT_COLUMNS],
|
| 1243 |
-
json.dumps(_with_result_configs(None, record.get('result_configs', []))),
|
| 1244 |
-
record.get('type', 'project'), record.get('listed', True))
|
| 1245 |
-
con.execute(f'INSERT INTO {self.t(PROJECTS)} ({", ".join(columns)}) '
|
| 1246 |
-
f'VALUES ({_marks(len(PROJECT_COLUMNS))}, %s::jsonb, %s, %s)', values)
|
| 1247 |
-
self._insert_targets(con, record['id'], pairs, stamp)
|
| 1248 |
-
self._replace_members(con, record['id'], record.get('contributors', []), [], stamp)
|
| 1249 |
-
|
| 1250 |
def insert(self, record, pairs, stamp):
|
| 1251 |
with self._write() as con:
|
| 1252 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1253 |
return self._one(con, record['id'])
|
| 1254 |
|
| 1255 |
def get(self, project_id):
|
|
@@ -1261,8 +1198,6 @@ class PgProjectStore:
|
|
| 1261 |
def update(self, project_id, changes, stamp):
|
| 1262 |
with self._write() as con:
|
| 1263 |
current = self._one(con, project_id, lock=True)
|
| 1264 |
-
if current['type'] == 'pdp' and set(changes) & {'progress', 'status', 'completed_on'}:
|
| 1265 |
-
raise ValueError('PDP item stages control project progress and completion.')
|
| 1266 |
candidate = _fields(changes, previous=current)
|
| 1267 |
if 'contributors' in changes:
|
| 1268 |
candidate['contributors'] = _member_values(changes['contributors'], self.tenant_slug)
|
|
@@ -1278,25 +1213,23 @@ class PgProjectStore:
|
|
| 1278 |
self._replace_members(con, project_id, candidate['contributors'], current['contributors'], stamp)
|
| 1279 |
return self._one(con, project_id)
|
| 1280 |
|
| 1281 |
-
def _delete(self, con, project_id):
|
| 1282 |
-
for table in (MEMBERS, TARGETS):
|
| 1283 |
-
con.execute(f'DELETE FROM {self.t(table)} WHERE tenant = %s AND project_id = %s',
|
| 1284 |
-
(self.tenant_slug, project_id))
|
| 1285 |
-
row = con.execute(f'DELETE FROM {self.t(PROJECTS)} WHERE tenant = %s AND id = %s RETURNING id',
|
| 1286 |
-
(self.tenant_slug, project_id)).fetchone()
|
| 1287 |
-
if row is None:
|
| 1288 |
-
raise KeyError('Project not found.')
|
| 1289 |
-
|
| 1290 |
def delete(self, project_id):
|
| 1291 |
with self._write() as con:
|
| 1292 |
-
self.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1293 |
return {'deleted': project_id}
|
| 1294 |
|
| 1295 |
def list_all(self):
|
| 1296 |
with self._connection() as con:
|
| 1297 |
if not self._exists(con):
|
| 1298 |
return []
|
| 1299 |
-
rows = con.execute(self._select_sql(
|
| 1300 |
members=self._members_projection(con)),
|
| 1301 |
(self.tenant_slug,)).fetchall()
|
| 1302 |
return [_record(row) for row in rows]
|
|
@@ -1320,12 +1253,10 @@ class PgProjectStore:
|
|
| 1320 |
count = (f'(SELECT count(*) FROM {self.t(TARGETS)} AS c '
|
| 1321 |
'WHERE c.tenant = p.tenant AND c.project_id = p.id)')
|
| 1322 |
rows = con.execute(
|
| 1323 |
-
f'SELECT t.grain, t.key, {cols}, {count}, {self._rollup_projection()}, {members}
|
| 1324 |
-
f'{self._type_projection()}, {self._listed_projection()} '
|
| 1325 |
f'FROM {self.t(TARGETS)} AS t '
|
| 1326 |
f'JOIN {self.t(PROJECTS)} AS p ON p.tenant = t.tenant AND p.id = t.project_id '
|
| 1327 |
-
f'WHERE t.tenant = %s AND
|
| 1328 |
-
'ORDER BY p.created_at, p.id',
|
| 1329 |
(self.tenant_slug, *(v for pair in part for v in pair))).fetchall()
|
| 1330 |
for row in rows:
|
| 1331 |
out[(row[0], row[1])].append(_record(row[2:]))
|
|
|
|
| 26 |
COMPLETED = 'Completed'
|
| 27 |
CANCELED = 'Canceled'
|
| 28 |
PRIORITIES = ('No priority', 'Urgent', 'High', 'Medium', 'Low')
|
|
|
|
| 29 |
#: Health turns At risk this many days before the target date.
|
| 30 |
AT_RISK_DAYS = 7
|
| 31 |
CHUNK = 200
|
|
|
|
| 243 |
value = values[len(columns) + 3] if len(values) > len(columns) + 3 else None
|
| 244 |
members = json.loads(value) if isinstance(value, str) else value
|
| 245 |
result['contributors'] = sorted(members or [], key=lambda member: member['username'])
|
|
|
|
|
|
|
| 246 |
return result
|
| 247 |
|
| 248 |
|
|
|
|
| 748 |
return f'{shown}. Target list incomplete' if shown else 'Target list incomplete'
|
| 749 |
|
| 750 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 751 |
def _activity_entries(event, actor_names, task_names):
|
| 752 |
"""Project one log event into named activity entries."""
|
| 753 |
out = []
|
| 754 |
for index, (field, change) in enumerate((event.get('changes') or {}).items()):
|
| 755 |
before, after = change.get('before'), change.get('after')
|
|
|
|
| 756 |
if field == 'project':
|
| 757 |
action, field = ('created' if before is None else 'deleted'), None
|
| 758 |
elif field == 'targets':
|
|
|
|
| 775 |
'by': event['by'], 'byUsername': event['by'],
|
| 776 |
'byName': actor_names.get(event['by'], event['by']),
|
| 777 |
'action': action, 'field': field, 'from': shown[0], 'to': shown[1]})
|
|
|
|
|
|
|
| 778 |
return out
|
| 779 |
|
| 780 |
|
|
|
|
| 1095 |
notes TEXT, completed_on DATE, source_id TEXT,
|
| 1096 |
created_at TEXT NOT NULL, updated_at TEXT NOT NULL, created_by TEXT,
|
| 1097 |
rollup JSONB, impact JSONB,
|
|
|
|
|
|
|
| 1098 |
PRIMARY KEY (tenant, id), UNIQUE (tenant, source_id),
|
| 1099 |
CHECK ((status = '{COMPLETED}' AND completed_on IS NOT NULL)
|
| 1100 |
OR (status <> '{COMPLETED}' AND completed_on IS NULL)))''')
|
|
|
|
| 1112 |
# One row's lookup is one indexed read on this index.
|
| 1113 |
con.execute(f'CREATE INDEX IF NOT EXISTS "amazon_project_targets_entity" '
|
| 1114 |
f'ON {self.t(TARGETS)} (tenant, grain, key, project_id)')
|
| 1115 |
+
for field in ('rollup', 'impact'):
|
| 1116 |
+
con.execute(f'ALTER TABLE {self.t(PROJECTS)} ADD COLUMN IF NOT EXISTS {field} JSONB')
|
|
|
|
|
|
|
|
|
|
| 1117 |
|
| 1118 |
@contextmanager
|
| 1119 |
def _write(self):
|
|
|
|
| 1126 |
cols = ', '.join(f'p.{c}' for c in PROJECT_COLUMNS)
|
| 1127 |
count = (f'(SELECT count(*) FROM {self.t(TARGETS)} AS c '
|
| 1128 |
'WHERE c.tenant = p.tenant AND c.project_id = p.id)')
|
| 1129 |
+
return (f'SELECT {prefix}{cols}, {count}, {self._rollup_projection()}, {members} '
|
|
|
|
| 1130 |
f'FROM {self.t(PROJECTS)} AS p WHERE p.tenant = %s{where}')
|
| 1131 |
|
| 1132 |
def _rollup_projection(self):
|
| 1133 |
# Old schemas read blank figures until a guarded write adds the columns.
|
| 1134 |
return "to_jsonb(p)->'rollup', to_jsonb(p)->'impact'"
|
| 1135 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1136 |
def _members_projection(self, con):
|
| 1137 |
if con.execute('SELECT to_regclass(%s)', (self.t(MEMBERS),)).fetchone()[0] is None:
|
| 1138 |
return 'NULL'
|
|
|
|
| 1178 |
'WHERE tenant = %s AND id = %s',
|
| 1179 |
(stamp, json.dumps(impact), self.tenant_slug, project_id))
|
| 1180 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1181 |
def insert(self, record, pairs, stamp):
|
| 1182 |
with self._write() as con:
|
| 1183 |
+
columns = (*PROJECT_COLUMNS, 'impact')
|
| 1184 |
+
values = (*[record[c] for c in PROJECT_COLUMNS],
|
| 1185 |
+
json.dumps(_with_result_configs(None, record.get('result_configs', []))))
|
| 1186 |
+
con.execute(f'INSERT INTO {self.t(PROJECTS)} ({", ".join(columns)}) '
|
| 1187 |
+
f'VALUES ({_marks(len(columns) - 1)}, %s::jsonb)', values)
|
| 1188 |
+
self._insert_targets(con, record['id'], pairs, stamp)
|
| 1189 |
+
self._replace_members(con, record['id'], record['contributors'], [], stamp)
|
| 1190 |
return self._one(con, record['id'])
|
| 1191 |
|
| 1192 |
def get(self, project_id):
|
|
|
|
| 1198 |
def update(self, project_id, changes, stamp):
|
| 1199 |
with self._write() as con:
|
| 1200 |
current = self._one(con, project_id, lock=True)
|
|
|
|
|
|
|
| 1201 |
candidate = _fields(changes, previous=current)
|
| 1202 |
if 'contributors' in changes:
|
| 1203 |
candidate['contributors'] = _member_values(changes['contributors'], self.tenant_slug)
|
|
|
|
| 1213 |
self._replace_members(con, project_id, candidate['contributors'], current['contributors'], stamp)
|
| 1214 |
return self._one(con, project_id)
|
| 1215 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1216 |
def delete(self, project_id):
|
| 1217 |
with self._write() as con:
|
| 1218 |
+
con.execute(f'DELETE FROM {self.t(MEMBERS)} WHERE tenant = %s AND project_id = %s',
|
| 1219 |
+
(self.tenant_slug, project_id))
|
| 1220 |
+
con.execute(f'DELETE FROM {self.t(TARGETS)} WHERE tenant = %s AND project_id = %s',
|
| 1221 |
+
(self.tenant_slug, project_id))
|
| 1222 |
+
row = con.execute(f'DELETE FROM {self.t(PROJECTS)} WHERE tenant = %s AND id = %s RETURNING id',
|
| 1223 |
+
(self.tenant_slug, project_id)).fetchone()
|
| 1224 |
+
if row is None:
|
| 1225 |
+
raise KeyError('Project not found.')
|
| 1226 |
return {'deleted': project_id}
|
| 1227 |
|
| 1228 |
def list_all(self):
|
| 1229 |
with self._connection() as con:
|
| 1230 |
if not self._exists(con):
|
| 1231 |
return []
|
| 1232 |
+
rows = con.execute(self._select_sql(' ORDER BY p.created_at, p.id',
|
| 1233 |
members=self._members_projection(con)),
|
| 1234 |
(self.tenant_slug,)).fetchall()
|
| 1235 |
return [_record(row) for row in rows]
|
|
|
|
| 1253 |
count = (f'(SELECT count(*) FROM {self.t(TARGETS)} AS c '
|
| 1254 |
'WHERE c.tenant = p.tenant AND c.project_id = p.id)')
|
| 1255 |
rows = con.execute(
|
| 1256 |
+
f'SELECT t.grain, t.key, {cols}, {count}, {self._rollup_projection()}, {members} '
|
|
|
|
| 1257 |
f'FROM {self.t(TARGETS)} AS t '
|
| 1258 |
f'JOIN {self.t(PROJECTS)} AS p ON p.tenant = t.tenant AND p.id = t.project_id '
|
| 1259 |
+
f'WHERE t.tenant = %s AND ({match}) ORDER BY p.created_at, p.id',
|
|
|
|
| 1260 |
(self.tenant_slug, *(v for pair in part for v in pair))).fetchall()
|
| 1261 |
for row in rows:
|
| 1262 |
out[(row[0], row[1])].append(_record(row[2:]))
|
platform/core/amazon_rivals.py
CHANGED
|
@@ -1,11 +1,10 @@
|
|
| 1 |
-
"""Per-tenant PostgreSQL
|
| 2 |
|
| 3 |
Discovery reads the Amazon mirror through a cursor the caller opens: core imports no harness
|
| 4 |
module (verify_layering) and no App module (rule 13). Tables are created on the first write.
|
| 5 |
"""
|
| 6 |
|
| 7 |
from contextlib import contextmanager
|
| 8 |
-
from dataclasses import dataclass
|
| 9 |
from datetime import date, datetime, timedelta, timezone
|
| 10 |
import json
|
| 11 |
import re
|
|
@@ -20,6 +19,8 @@ RUNS = 'amazon_rival_runs'
|
|
| 20 |
WANTED = 'amazon_rival_wanted'
|
| 21 |
TABLES = (RIVALS, DISCOVERY, QUOTES, RUNS, WANTED)
|
| 22 |
|
|
|
|
|
|
|
| 23 |
#: Brand Analytics keeps the top 3 clicked ASINs of each search term.
|
| 24 |
TOP_CLICKED = 3
|
| 25 |
SOURCES = frozenset(('auto', 'added'))
|
|
@@ -28,35 +29,16 @@ CHUNK = 200
|
|
| 28 |
BA_TABLE = 'amzn_brand_analytics'
|
| 29 |
LISTINGS_TABLE = 'amzn_listings'
|
| 30 |
PRICING_TABLE = 'amzn_pricing'
|
| 31 |
-
SQP_TABLE = 'amzn_search_query_performance'
|
| 32 |
-
RANKS_TABLE = 'amzn_apify_rank_observations'
|
| 33 |
-
SNAPSHOTS_TABLE = 'amzn_competitor_snapshots'
|
| 34 |
-
IMAGES_TABLE = 'amzn_competitor_images'
|
| 35 |
-
POOL_SNAPSHOT_FIELDS = ('snapshot_date', 'brand', 'title', 'bullets', 'price', 'currency', 'prime', 'offers')
|
| 36 |
-
POOL_IMAGE_FIELDS = ('variant', 'link', 'width', 'height')
|
| 37 |
|
| 38 |
NO_TOP3 = "Not in any search term's top 3 clicked"
|
| 39 |
NO_WEEK = 'Brand Analytics has no dated weekly reports'
|
| 40 |
-
NO_MONTH = 'No completed monthly Search Query Performance report for this product yet.'
|
| 41 |
-
NO_MONEY = 'No searches with market money for this product yet.'
|
| 42 |
-
NO_KEYWORD_RANKS = "No keyword ranks for this product's searches yet."
|
| 43 |
-
NO_PRODUCT_TYPE = 'Product type not read yet. The next daily run reads it.'
|
| 44 |
-
WAITING_PRODUCT_TYPE = '{count} found, waiting for the daily read of their product type.'
|
| 45 |
NOT_OURS = 'Not an ASIN in your Amazon listings'
|
| 46 |
PARENT_ROLLS_UP = 'A Parent ASIN has no rival list. Its fields roll up its child ASINs.'
|
| 47 |
-
NO_POOL = 'No search report or keyword ranks for this product yet.'
|
| 48 |
ASIN_RE = re.compile(r'[A-Z0-9]{10}')
|
| 49 |
|
| 50 |
-
POOL_FIELDS = {
|
| 51 |
-
RIVALS: {'origin': ('TEXT', "'click'"), 'money_won': ('DOUBLE PRECISION', '0'),
|
| 52 |
-
'terms_won': ('INTEGER', '0'), 'term_rank': ('INTEGER', 'NULL'),
|
| 53 |
-
'eligible': ('BOOLEAN', 'TRUE')},
|
| 54 |
-
DISCOVERY: {'month': ('DATE', 'NULL'), 'keyword_ranks_on': ('DATE', 'NULL'),
|
| 55 |
-
'product_type': ('TEXT', 'NULL'), 'held': ('INTEGER', '0')},
|
| 56 |
-
}
|
| 57 |
RIVAL_COLUMNS = ('asin', 'rival_asin', 'source', 'removed', 'click_share', 'term', 'title', 'week',
|
| 58 |
-
'added_by', 'updated_at'
|
| 59 |
-
DISCOVERY_COLUMNS = ('asin', 'week', 'discovered_at', 'cause', 'auto_count'
|
| 60 |
QUOTE_COLUMNS = ('asin', 'quote_on', 'offer_price', 'buy_box_price', 'sales_rank', 'display_group',
|
| 61 |
'offer_cause', 'buy_box_cause', 'rank_cause', 'offers', 'currency', 'run_id', 'quoted_at')
|
| 62 |
RUN_COLUMNS = ('id', 'started_at', 'progress', 'progress_at', 'finished_at', 'asin_total', 'priced',
|
|
@@ -193,355 +175,96 @@ def children(cur, parent):
|
|
| 193 |
return [r[0] for r in rows if r[0]]
|
| 194 |
|
| 195 |
|
| 196 |
-
def
|
| 197 |
-
"""
|
| 198 |
-
required = {'asin', 'search_query_data_search_query', 'start_date', 'end_date',
|
| 199 |
-
'purchase_data_total_purchase_count', 'purchase_data_total_median_purchase_price_amount'}
|
| 200 |
-
if not required <= _columns(cur, SQP_TABLE):
|
| 201 |
-
return ('SELECT NULL::VARCHAR AS asin, NULL::VARCHAR AS term, NULL::DATE AS month, '
|
| 202 |
-
'NULL::DOUBLE AS market_money WHERE FALSE')
|
| 203 |
-
return f'''
|
| 204 |
-
WITH reports AS (
|
| 205 |
-
SELECT upper(trim(CAST(asin AS VARCHAR))) AS asin,
|
| 206 |
-
lower(trim(CAST(search_query_data_search_query AS VARCHAR))) AS term,
|
| 207 |
-
TRY_CAST(start_date AS DATE) AS start_on, TRY_CAST(end_date AS DATE) AS end_on,
|
| 208 |
-
TRY_CAST(purchase_data_total_purchase_count AS DOUBLE) *
|
| 209 |
-
TRY_CAST(purchase_data_total_median_purchase_price_amount AS DOUBLE) AS market_money
|
| 210 |
-
FROM {SQP_TABLE}
|
| 211 |
-
WHERE date_diff('day', TRY_CAST(start_date AS DATE), TRY_CAST(end_date AS DATE)) > 7
|
| 212 |
-
AND TRY_CAST(end_date AS DATE) < CURRENT_DATE),
|
| 213 |
-
latest AS (
|
| 214 |
-
SELECT * FROM reports QUALIFY end_on = max(end_on) OVER (PARTITION BY asin))
|
| 215 |
-
SELECT asin, term, CAST(date_trunc('month', start_on) AS DATE) AS month,
|
| 216 |
-
max(market_money) AS market_money
|
| 217 |
-
FROM latest GROUP BY asin, term, month'''
|
| 218 |
-
|
| 219 |
-
|
| 220 |
-
def _click_sql(cur):
|
| 221 |
-
"""Normalized dated top-three clicks, with percentage shares kept in their source units."""
|
| 222 |
cols = _columns(cur, BA_TABLE)
|
| 223 |
-
|
| 224 |
-
|
| 225 |
-
return ('SELECT NULL::VARCHAR AS term, NULL::VARCHAR AS rival, NULL::DOUBLE AS share, '
|
| 226 |
-
'NULL::VARCHAR AS title, NULL::INTEGER AS term_rank, NULL::DATE AS week WHERE FALSE')
|
| 227 |
-
title = 'CAST(clicked_item_name AS VARCHAR)' if 'clicked_item_name' in cols else 'NULL::VARCHAR'
|
| 228 |
-
return f'''SELECT lower(trim(CAST(search_term AS VARCHAR))) AS term,
|
| 229 |
-
upper(trim(CAST(clicked_asin AS VARCHAR))) AS rival,
|
| 230 |
-
TRY_CAST(click_share AS DOUBLE) AS share, {title} AS title,
|
| 231 |
-
TRY_CAST(click_share_rank AS INTEGER) AS term_rank, {_week_sql()} AS week
|
| 232 |
-
FROM {BA_TABLE} WHERE {weekly_period_sql()}
|
| 233 |
-
AND TRY_CAST(click_share_rank AS INTEGER) BETWEEN 1 AND {TOP_CLICKED}'''
|
| 234 |
-
|
| 235 |
-
|
| 236 |
-
def _rank_rows_sql(cur):
|
| 237 |
-
"""Raw Keyword ranks observations include rivals absent from the own-placement view."""
|
| 238 |
-
if not {'keyword', 'asin', 'date', 'organic_rank', 'is_sponsored'} <= _columns(cur, RANKS_TABLE):
|
| 239 |
-
return ('SELECT NULL::VARCHAR AS term, NULL::VARCHAR AS rival, NULL::DATE AS observed_on, '
|
| 240 |
-
'NULL::INTEGER AS term_rank, NULL::BOOLEAN AS sponsored WHERE FALSE')
|
| 241 |
-
return f'''SELECT lower(trim(CAST(keyword AS VARCHAR))) AS term,
|
| 242 |
-
upper(trim(CAST(asin AS VARCHAR))) AS rival, TRY_CAST(date AS DATE) AS observed_on,
|
| 243 |
-
TRY_CAST(organic_rank AS INTEGER) AS term_rank,
|
| 244 |
-
TRY_CAST(is_sponsored AS BOOLEAN) AS sponsored FROM {RANKS_TABLE}'''
|
| 245 |
-
|
| 246 |
-
|
| 247 |
-
def _search_dates(cur, targets):
|
| 248 |
-
rows = _read(cur, f'''
|
| 249 |
-
WITH money AS ({_money_sql(cur)}), observations AS ({_rank_rows_sql(cur)})
|
| 250 |
-
SELECT m.asin, max(m.month), max(o.observed_on), count(DISTINCT m.term)
|
| 251 |
-
FILTER (WHERE m.market_money > 0)
|
| 252 |
-
FROM money m LEFT JOIN observations o ON o.term = m.term AND m.market_money > 0
|
| 253 |
-
WHERE m.asin IN {_in(targets)} GROUP BY m.asin''', targets)
|
| 254 |
-
return {a: dict(month=_day(m), keyword_ranks_on=_day(k), money_terms=n) for a, m, k, n in rows}
|
| 255 |
-
|
| 256 |
-
|
| 257 |
-
def _newest_snapshots_sql(cur, fields=('product_type', 'brand')):
|
| 258 |
-
if not {'asin', 'snapshot_date', 'product_type', 'brand'} <= _columns(cur, SNAPSHOTS_TABLE):
|
| 259 |
-
return 'SELECT NULL::VARCHAR AS asin, ' + ', '.join(f'NULL::VARCHAR AS {f}' for f in fields) + ' WHERE FALSE'
|
| 260 |
-
return (f'SELECT upper(trim(asin)) AS asin, {", ".join(fields)} FROM {SNAPSHOTS_TABLE} '
|
| 261 |
-
'QUALIFY row_number() OVER (PARTITION BY upper(trim(asin)) ORDER BY snapshot_date DESC) = 1')
|
| 262 |
-
|
| 263 |
-
|
| 264 |
-
def _brand(value):
|
| 265 |
-
return ''.join((value or '').casefold().split())
|
| 266 |
-
|
| 267 |
-
|
| 268 |
-
def _same_type(target_type, rival_type):
|
| 269 |
-
return bool(target_type and rival_type and target_type == rival_type)
|
| 270 |
-
|
| 271 |
-
|
| 272 |
-
def _other_brand(brand, own_brands):
|
| 273 |
-
return not _brand(brand) or _brand(brand) not in own_brands
|
| 274 |
-
|
| 275 |
-
|
| 276 |
-
@dataclass
|
| 277 |
-
class PoolFacts:
|
| 278 |
-
"""Newest ASIN facts and all historical own brands determine pool eligibility."""
|
| 279 |
-
|
| 280 |
-
latest: dict
|
| 281 |
-
own_brands: set
|
| 282 |
-
own_asins: set
|
| 283 |
-
|
| 284 |
-
def product_type(self, target):
|
| 285 |
-
return self.latest.get(target, {}).get('product_type')
|
| 286 |
-
|
| 287 |
-
def eligible(self, target, rival):
|
| 288 |
-
fact = self.latest.get(rival, {})
|
| 289 |
-
return (rival not in self.own_asins and _same_type(self.product_type(target), fact.get('product_type'))
|
| 290 |
-
and _other_brand(fact.get('brand'), self.own_brands))
|
| 291 |
-
|
| 292 |
-
|
| 293 |
-
def _pool_facts(cur, values, own, prefetched=None, candidate_rows=()):
|
| 294 |
-
latest = {r[1]: dict(product_type=r[-2], brand=r[-1]) for r in candidate_rows}
|
| 295 |
-
for part in _chunks(sorted(values - latest.keys())):
|
| 296 |
-
rows = _read(cur, f'SELECT asin, product_type, brand FROM ({_newest_snapshots_sql(cur)}) '
|
| 297 |
-
f'WHERE asin IN {_in(part)}', part)
|
| 298 |
-
latest.update({a: dict(product_type=t, brand=b) for a, t, b in rows})
|
| 299 |
-
brands = set()
|
| 300 |
-
if {'asin', 'brand'} <= _columns(cur, SNAPSHOTS_TABLE):
|
| 301 |
-
rows = _read(cur, f'SELECT DISTINCT s.brand FROM {SNAPSHOTS_TABLE} s '
|
| 302 |
-
f'JOIN {LISTINGS_TABLE} l ON upper(trim(s.asin)) = upper(trim(l.asin1))')
|
| 303 |
-
brands.update(_brand(row[0]) for row in rows if _brand(row[0]))
|
| 304 |
-
latest.update(prefetched or {})
|
| 305 |
-
brands.update(_brand(latest[a].get('brand')) for a in own if a in latest and _brand(latest[a].get('brand')))
|
| 306 |
-
return PoolFacts(latest, brands, own)
|
| 307 |
-
|
| 308 |
-
|
| 309 |
-
def _candidates(cur, targets, week):
|
| 310 |
-
"""Every non-own clicked or organic page-one ASIN on each target's monthly money searches."""
|
| 311 |
sql = f'''
|
| 312 |
-
WITH
|
| 313 |
-
|
| 314 |
-
|
| 315 |
-
|
| 316 |
-
|
| 317 |
-
|
| 318 |
-
|
| 319 |
-
|
| 320 |
-
|
| 321 |
-
SELECT term, rival, min(term_rank) AS term_rank FROM newest_observations
|
| 322 |
-
WHERE sponsored = FALSE GROUP BY term, rival),
|
| 323 |
-
placements AS (
|
| 324 |
-
SELECT coalesce(c.term, p.term) AS term, coalesce(c.rival, p.rival) AS rival,
|
| 325 |
-
c.share, c.title, coalesce(c.term_rank, p.term_rank) AS term_rank,
|
| 326 |
-
c.term IS NOT NULL AS clicked, p.term IS NOT NULL AS page1
|
| 327 |
-
FROM click c FULL OUTER JOIN page1 p ON p.term = c.term AND p.rival = c.rival),
|
| 328 |
hits AS (
|
| 329 |
-
SELECT
|
| 330 |
-
|
| 331 |
-
|
| 332 |
-
|
| 333 |
-
|
| 334 |
-
|
| 335 |
-
SELECT asin, rival, max(share) AS share,
|
| 336 |
-
|
| 337 |
-
|
| 338 |
-
|
| 339 |
-
|
| 340 |
-
|
| 341 |
-
|
| 342 |
-
|
| 343 |
-
ORDER BY money_won DESC, term_rank, term) = 1)
|
| 344 |
-
SELECT t.asin, t.rival, t.share, r.term, r.title, t.origin, t.money_won, t.terms_won, r.term_rank,
|
| 345 |
-
s.product_type, s.brand
|
| 346 |
-
FROM totals t JOIN representative r USING (asin, rival)
|
| 347 |
-
LEFT JOIN ({_newest_snapshots_sql(cur)}) s ON s.asin = t.rival
|
| 348 |
-
ORDER BY t.asin, t.money_won DESC, t.rival'''
|
| 349 |
-
return _read(cur, sql, (week, *targets))
|
| 350 |
|
| 351 |
|
| 352 |
# --------------------------------------------------------------------------- discovery
|
| 353 |
|
| 354 |
-
def discover(rt, cur, asins=None
|
| 355 |
-
"""
|
|
|
|
|
|
|
| 356 |
backend = _backend(rt)
|
| 357 |
requested = None if asins is None else _asins(asins)
|
| 358 |
targets = sorted(our_asins(cur, requested))
|
| 359 |
stamp = _now()
|
| 360 |
discovery = {}
|
| 361 |
for missing in sorted(set(requested or ()) - set(targets)):
|
| 362 |
-
discovery[missing] = dict(asin=missing, week=None, discovered_at=stamp, cause=NOT_OURS,
|
| 363 |
-
auto_count=0, month=None, keyword_ranks_on=None, product_type=None, held=0)
|
| 364 |
week = newest_week(cur) if targets else None
|
| 365 |
-
|
| 366 |
-
|
| 367 |
-
|
| 368 |
-
|
| 369 |
-
|
| 370 |
-
|
| 371 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 372 |
for target in targets:
|
| 373 |
-
count =
|
| 374 |
-
|
| 375 |
-
|
| 376 |
-
|
| 377 |
-
discovery[target] = dict(asin=target, week=week, discovered_at=stamp, cause=cause, auto_count=count,
|
| 378 |
-
month=meta['month'], keyword_ranks_on=meta['keyword_ranks_on'],
|
| 379 |
-
product_type=facts.product_type(target), held=held.get(target, 0))
|
| 380 |
-
ready = [t for t in targets if facts.product_type(t)]
|
| 381 |
-
backend.replace_auto(ready, [row for t in ready for row in found.get(t, ())], list(discovery.values()))
|
| 382 |
-
backend.set_eligibility(stored, facts)
|
| 383 |
-
members = backend.rivals_of(targets)
|
| 384 |
-
pools = {target: len(members.get(target, ())) for target in targets}
|
| 385 |
return {'asins': len(targets), 'rivals': sum(len(v) for v in found.values()), 'week': week,
|
| 386 |
-
'
|
| 387 |
-
'changed': tuple(t for t in targets if set(before.get(t, ())) != set(members.get(t, ()))),
|
| 388 |
-
'causes': {k: v['cause'] for k, v in discovery.items() if v['cause']}}
|
| 389 |
-
|
| 390 |
-
|
| 391 |
-
def _eligible_candidates(rows, stored, facts, week, stamp):
|
| 392 |
-
kept = {(r['asin'], r['rival_asin']) for r in stored if r['removed'] or r['source'] == 'added'}
|
| 393 |
-
candidates = {(r[0], r[1]) for r in rows} | {
|
| 394 |
-
(r['asin'], r['rival_asin']) for r in stored if r['source'] == 'added' and not r['removed']}
|
| 395 |
-
removed = {(r['asin'], r['rival_asin']) for r in stored if r['removed']}
|
| 396 |
-
held = {}
|
| 397 |
-
for target, rival in candidates - removed:
|
| 398 |
-
if (rival not in facts.own_asins and not facts.product_type(rival)
|
| 399 |
-
and _other_brand(facts.latest.get(rival, {}).get('brand'), facts.own_brands)):
|
| 400 |
-
held[target] = held.get(target, 0) + 1
|
| 401 |
-
found = {}
|
| 402 |
-
for target, rival, share, term, title, origin, money, terms, term_rank, _type, _name in rows:
|
| 403 |
-
if (target, rival) in kept or not facts.eligible(target, rival):
|
| 404 |
-
continue
|
| 405 |
-
found.setdefault(target, []).append(dict(
|
| 406 |
-
asin=target, rival_asin=rival, source='auto', removed=False, eligible=True,
|
| 407 |
-
click_share=None if share is None else float(share), term=term, title=title,
|
| 408 |
-
week=week, added_by=None, updated_at=stamp, origin=origin,
|
| 409 |
-
money_won=float(money), terms_won=int(terms), term_rank=term_rank))
|
| 410 |
-
return found, held
|
| 411 |
-
|
| 412 |
-
|
| 413 |
-
def _discovery_cause(week, meta, count):
|
| 414 |
-
if meta['month'] is None:
|
| 415 |
-
return NO_MONTH
|
| 416 |
-
if not meta['money_terms']:
|
| 417 |
-
return NO_MONEY
|
| 418 |
-
if meta['keyword_ranks_on'] is None:
|
| 419 |
-
return NO_KEYWORD_RANKS
|
| 420 |
-
if week is None:
|
| 421 |
-
return NO_WEEK
|
| 422 |
-
return None if count else NO_TOP3
|
| 423 |
-
|
| 424 |
-
|
| 425 |
-
def _pool_cause(product_type, held, week, meta, count):
|
| 426 |
-
if not product_type:
|
| 427 |
-
return NO_PRODUCT_TYPE
|
| 428 |
-
if held:
|
| 429 |
-
return WAITING_PRODUCT_TYPE.format(count=held)
|
| 430 |
-
return _discovery_cause(week, meta, count)
|
| 431 |
|
| 432 |
|
| 433 |
def stale(rt, cur, target):
|
| 434 |
-
"""True when
|
| 435 |
row = _backend(rt).discovery([target]).get(target)
|
| 436 |
if row is None:
|
| 437 |
return True
|
| 438 |
-
|
| 439 |
-
|
| 440 |
-
if row['product_type'] != product_type or (not product_type and row['cause'] != NO_PRODUCT_TYPE):
|
| 441 |
-
return True
|
| 442 |
-
newest = {'week': newest_week(cur), **(_search_dates(cur, [target]).get(target) or {})}
|
| 443 |
-
return any(newest.get(key) is not None and
|
| 444 |
-
(row.get(key) is None or row[key] < newest[key])
|
| 445 |
-
for key in ('week', 'month', 'keyword_ranks_on'))
|
| 446 |
|
| 447 |
|
| 448 |
# --------------------------------------------------------------------------- doors
|
| 449 |
|
| 450 |
def rivals(rt, target):
|
| 451 |
-
"""One ASIN's rivals: hand-added first, then
|
| 452 |
target = asin(target)
|
| 453 |
backend = _backend(rt)
|
| 454 |
rows = backend.rivals(target)
|
| 455 |
found = backend.discovery([target]).get(target)
|
| 456 |
return {'asin': target, 'rivals': rows, 'removed': backend.removed_count(target),
|
| 457 |
'week': found['week'] if found else None,
|
| 458 |
-
'month': found['month'] if found else None,
|
| 459 |
-
'keywordRanksOn': found['keyword_ranks_on'] if found else None,
|
| 460 |
'discoveredAt': found['discovered_at'] if found else None,
|
| 461 |
-
'cause': found['cause'] if found else None
|
| 462 |
-
'productType': found['product_type'] if found else None,
|
| 463 |
-
'held': found['held'] if found else 0}
|
| 464 |
-
|
| 465 |
-
|
| 466 |
-
def pool(rt, cur, target):
|
| 467 |
-
"""Read eligible pool rows through the caller's mirror cursor without writes or provider reads."""
|
| 468 |
-
target = asin(target)
|
| 469 |
-
backend = _backend(rt)
|
| 470 |
-
if cur is None:
|
| 471 |
-
return {**rivals(rt, target), 'rivals': [], 'competitors': [], 'dataAsOf': None, 'cause': NO_POOL}
|
| 472 |
-
own = our_asins(cur)
|
| 473 |
-
week = newest_week(cur)
|
| 474 |
-
sources = _candidates(cur, [target], week) if target in own else []
|
| 475 |
-
stored = backend.pool_rows([target])
|
| 476 |
-
facts = _pool_facts(cur, {target} | {r[1] for r in sources} | {r['rival_asin'] for r in stored},
|
| 477 |
-
own, candidate_rows=sources)
|
| 478 |
-
found, held = _eligible_candidates(sources, stored, facts, week, _now())
|
| 479 |
-
if facts.product_type(target):
|
| 480 |
-
rows = found.get(target, []) + [dict(r, eligible=True) for r in stored
|
| 481 |
-
if r['source'] == 'added' and not r['removed']
|
| 482 |
-
and facts.eligible(target, r['rival_asin'])]
|
| 483 |
-
else:
|
| 484 |
-
rows = [r for r in stored if r['eligible'] and not r['removed'] and r['rival_asin'] not in own
|
| 485 |
-
and _other_brand(facts.latest.get(r['rival_asin'], {}).get('brand'), facts.own_brands)]
|
| 486 |
-
if target not in own:
|
| 487 |
-
rows = []
|
| 488 |
-
averages = backend.rank_averages([r['rival_asin'] for r in rows])
|
| 489 |
-
rows = [{**r, **averages.get(r['rival_asin'], {'rank_average': None, 'rank_days': 0})} for r in rows]
|
| 490 |
-
meta = _search_dates(cur, [target]).get(target) or dict(month=None, keyword_ranks_on=None, money_terms=0)
|
| 491 |
-
cause = _pool_cause(facts.product_type(target), held.get(target, 0), week, meta, len(rows))
|
| 492 |
-
rows = sorted(rows, key=_pool_order)
|
| 493 |
-
snapshots = _pool_snapshots(cur, [target, *(r['rival_asin'] for r in rows)])
|
| 494 |
-
images = _pool_images(cur, [r['rival_asin'] for r in rows])
|
| 495 |
-
dates = [s['snapshot_date'] for s in snapshots.values() if s['snapshot_date'] is not None]
|
| 496 |
-
return {**rivals(rt, target), 'rivals': rows, 'week': week,
|
| 497 |
-
'month': meta['month'], 'keywordRanksOn': meta['keyword_ranks_on'],
|
| 498 |
-
'productType': facts.product_type(target), 'held': held.get(target, 0),
|
| 499 |
-
'cause': cause if target in own else NOT_OURS, 'dataAsOf': max(dates, default=None),
|
| 500 |
-
'competitors': [_pool_competitor(r, snapshots.get(r['rival_asin'], {}),
|
| 501 |
-
images.get(r['rival_asin'], [])) for r in rows]}
|
| 502 |
-
|
| 503 |
-
|
| 504 |
-
def _pool_snapshots(cur, values):
|
| 505 |
-
snapshots = {}
|
| 506 |
-
for part in _chunks(sorted(set(values))):
|
| 507 |
-
rows = _read(cur, f'SELECT * FROM ({_newest_snapshots_sql(cur, POOL_SNAPSHOT_FIELDS)}) '
|
| 508 |
-
f'WHERE asin IN {_in(part)}', part)
|
| 509 |
-
for target, *fields in rows:
|
| 510 |
-
snapshot = dict(zip(POOL_SNAPSHOT_FIELDS, fields))
|
| 511 |
-
snapshot['snapshot_date'] = _day(snapshot['snapshot_date'])
|
| 512 |
-
snapshot['bullets'] = json.loads(snapshot['bullets'] or '[]')
|
| 513 |
-
snapshots[target] = snapshot
|
| 514 |
-
return snapshots
|
| 515 |
-
|
| 516 |
-
|
| 517 |
-
def _pool_images(cur, values):
|
| 518 |
-
"""Resolve only the newest snapshot's variants against changed-link history."""
|
| 519 |
-
images = {}
|
| 520 |
-
if not {'asin', *POOL_IMAGE_FIELDS} <= _columns(cur, IMAGES_TABLE):
|
| 521 |
-
return images
|
| 522 |
-
for part in _chunks(values):
|
| 523 |
-
rows = _read(cur, f'''
|
| 524 |
-
SELECT s.asin, i.variant, i.link, i.width, i.height
|
| 525 |
-
FROM ({_newest_snapshots_sql(cur, ('image_variants',))}) s,
|
| 526 |
-
json_each(s.image_variants) v, {IMAGES_TABLE} i
|
| 527 |
-
WHERE s.asin IN {_in(part)} AND i.asin = s.asin
|
| 528 |
-
AND i.variant = json_extract_string(v.value, '$.variant')
|
| 529 |
-
AND i.link = json_extract_string(v.value, '$.link')
|
| 530 |
-
ORDER BY s.asin, CAST(v.key AS INTEGER)''', part)
|
| 531 |
-
for target, *fields in rows:
|
| 532 |
-
images.setdefault(target, []).append(dict(zip(POOL_IMAGE_FIELDS, fields)))
|
| 533 |
-
return images
|
| 534 |
-
|
| 535 |
-
|
| 536 |
-
def _pool_competitor(row, snapshot, images):
|
| 537 |
-
days = row['rank_days']
|
| 538 |
-
return {'asin': row['rival_asin'], 'brand': snapshot.get('brand'), 'title': snapshot.get('title'),
|
| 539 |
-
'origin': row['origin'], 'moneyWon': row['money_won'], 'termsWon': row['terms_won'],
|
| 540 |
-
'term': row['term'], 'termRank': row['term_rank'], 'bestSellerRank': row['rank_average'],
|
| 541 |
-
'rankDays': days, 'rankCause': f'Rank average needs 7 days of ranks. {days} so far.' if days < 7 else None,
|
| 542 |
-
'price': snapshot.get('price'), 'currency': snapshot.get('currency'), 'prime': snapshot.get('prime'),
|
| 543 |
-
'offers': snapshot.get('offers'), 'imageCount': len(images), 'images': images,
|
| 544 |
-
'bullets': snapshot.get('bullets', []), 'snapshotOn': snapshot.get('snapshot_date')}
|
| 545 |
|
| 546 |
|
| 547 |
def add(rt, target, rival, *, user, title=None):
|
|
@@ -549,10 +272,9 @@ def add(rt, target, rival, *, user, title=None):
|
|
| 549 |
target, rival = asin(target), asin(rival, 'rival')
|
| 550 |
if rival == target:
|
| 551 |
raise ValueError('An ASIN cannot be its own rival.')
|
| 552 |
-
return _backend(rt).add(dict(asin=target, rival_asin=rival, source='added', removed=False,
|
| 553 |
click_share=None, term=None, title=title, week=None,
|
| 554 |
-
added_by=str(user or '') or None, updated_at=_now()
|
| 555 |
-
origin='added', money_won=0, terms_won=0, term_rank=None))
|
| 556 |
|
| 557 |
|
| 558 |
def remove(rt, target, rival, *, user):
|
|
@@ -616,19 +338,6 @@ def runs(rt, limit=10):
|
|
| 616 |
return _backend(rt).runs(int(limit))
|
| 617 |
|
| 618 |
|
| 619 |
-
def rank_average(rt, asins):
|
| 620 |
-
"""Mean daily rank in each ASIN's newest display group, requiring seven of its newest 30 days."""
|
| 621 |
-
values = _asins(asins)
|
| 622 |
-
found = _backend(rt).rank_averages(values) if values else {}
|
| 623 |
-
return {a: (found.get(a) or {}).get('rank_average') for a in values}
|
| 624 |
-
|
| 625 |
-
|
| 626 |
-
def _pool_order(row):
|
| 627 |
-
average = row.get('rank_average')
|
| 628 |
-
return (-float(row.get('money_won') or 0),
|
| 629 |
-
average is None, average if average is not None else 0, row['rival_asin'])
|
| 630 |
-
|
| 631 |
-
|
| 632 |
# --------------------------------------------------------------------------- the daily quotes run
|
| 633 |
|
| 634 |
#: The run row gets a progress marker after each block of this many ASINs.
|
|
@@ -642,8 +351,7 @@ MAX_BACKOFF_SECONDS = 300.0
|
|
| 642 |
#: An unfinished run, or a finished run with a cause, is retried after this many hours.
|
| 643 |
RETRY_RUN_HOURS = 3
|
| 644 |
|
| 645 |
-
READS = {'offers': 'offers', 'summaries': 'Buy Box price', '
|
| 646 |
-
FACT_COUNT_COLUMNS = {'title': 'title', 'product_type': 'product_type', 'image': 'image', 'rank': 'sales_rank'}
|
| 647 |
NO_MIRROR = 'The Amazon data is not on this worker yet, so no ASIN was quoted.'
|
| 648 |
NO_OWN_OFFER = 'Amazon returned no offer of yours on this ASIN.'
|
| 649 |
NO_BUY_BOX = 'Amazon returned no Buy Box offer for this ASIN.'
|
|
@@ -741,7 +449,7 @@ def _summary_quote(response):
|
|
| 741 |
|
| 742 |
|
| 743 |
def _keyed(kind, got):
|
| 744 |
-
if kind == '
|
| 745 |
return {str(k).upper(): v for k, v in (got or {}).items()}
|
| 746 |
if kind == 'offers':
|
| 747 |
return {str((((r or {}).get('body') or {}).get('payload') or {}).get('ASIN') or '').upper(): r
|
|
@@ -781,16 +489,7 @@ def _read_batches(kind, fn, asins, *, sleep, log, counts):
|
|
| 781 |
attempt = 0
|
| 782 |
while True:
|
| 783 |
try:
|
| 784 |
-
|
| 785 |
-
answers.update(got)
|
| 786 |
-
if kind == 'facts':
|
| 787 |
-
for target, fact in got.items():
|
| 788 |
-
if fact.get('cause'):
|
| 789 |
-
causes[target] = fact['cause']
|
| 790 |
-
if '(429)' in fact['cause']:
|
| 791 |
-
counts['throttled'] += 1
|
| 792 |
-
elif 'Amazon refused' in fact['cause']:
|
| 793 |
-
counts['refused'] += 1
|
| 794 |
break
|
| 795 |
except Exception as exc: # noqa: BLE001
|
| 796 |
if _throttled(exc) and attempt < THROTTLE_RETRIES:
|
|
@@ -806,19 +505,14 @@ def _read_batches(kind, fn, asins, *, sleep, log, counts):
|
|
| 806 |
return answers, causes
|
| 807 |
|
| 808 |
|
| 809 |
-
def _quote_block(block, own, readers, seller_id, quote_on, run_id, sleep, log, counts
|
| 810 |
-
prefetched=None, fact_causes=None):
|
| 811 |
"""One quote row per ASIN of `block`, each with a value or a cause for every field it owns."""
|
| 812 |
mine = [a for a in block if a in own]
|
| 813 |
others = [a for a in block if a not in own]
|
| 814 |
offered, offer_causes = _read_batches('offers', readers['offers'], mine, sleep=sleep, log=log, counts=counts)
|
| 815 |
summed, sum_causes = _read_batches('summaries', readers['summaries'], others, sleep=sleep, log=log,
|
| 816 |
counts=counts)
|
| 817 |
-
|
| 818 |
-
facts, rank_causes = _read_batches('facts', readers['facts'], [a for a in block if a not in cached],
|
| 819 |
-
sleep=sleep, log=log, counts=counts)
|
| 820 |
-
facts.update({a: cached[a] for a in block if a in cached})
|
| 821 |
-
rank_causes.update(fact_causes or {})
|
| 822 |
stamp = _now()
|
| 823 |
rows = []
|
| 824 |
for target in block:
|
|
@@ -837,31 +531,20 @@ def _quote_block(block, own, readers, seller_id, quote_on, run_id, sleep, log, c
|
|
| 837 |
row.update(_summary_quote(summed[target]))
|
| 838 |
else:
|
| 839 |
row['buy_box_cause'] = NO_PRICE
|
| 840 |
-
rank =
|
| 841 |
value = rank.get('rank') if isinstance(rank.get('rank'), int) and not isinstance(rank.get('rank'), bool) else None
|
| 842 |
row.update(sales_rank=value, display_group=rank.get('group') if value is not None else None,
|
| 843 |
-
rank_cause=None if value is not None else
|
| 844 |
rows.append(row)
|
| 845 |
-
return rows
|
| 846 |
-
|
| 847 |
-
|
| 848 |
-
def _snapshot(quote, facts, own):
|
| 849 |
-
"""C3 facts with price and rank taken from the exact persisted quote row."""
|
| 850 |
-
return {**{key: facts.get(key) for key in ('product_type', 'brand', 'title', 'bullets',
|
| 851 |
-
'image_variants', 'prime', 'offers', 'rank_cat', 'image')},
|
| 852 |
-
'asin': quote['asin'], 'snapshot_date': quote['quote_on'],
|
| 853 |
-
'price': quote['offer_price' if own else 'buy_box_price'], 'currency': quote['currency'],
|
| 854 |
-
'sales_rank': quote['sales_rank'], 'display_group': quote['display_group']}
|
| 855 |
|
| 856 |
|
| 857 |
-
def _tally(counts, rows, own
|
| 858 |
for row in rows:
|
| 859 |
counts['priced'] += int((row['offer_price'] if row['asin'] in own else None) is not None
|
| 860 |
or row['buy_box_price'] is not None)
|
| 861 |
counts['ranked'] += int(row['sales_rank'] is not None)
|
| 862 |
counts['caused'] += int(bool(row['offer_cause'] or row['buy_box_cause'] or row['rank_cause']))
|
| 863 |
-
for key, field in FACT_COUNT_COLUMNS.items():
|
| 864 |
-
counts['facts'][key] += sum(row.get(field) is not None and row[field] != '' for row in snapshots)
|
| 865 |
|
| 866 |
|
| 867 |
def _changed(before, row):
|
|
@@ -869,47 +552,35 @@ def _changed(before, row):
|
|
| 869 |
return old is None or any(old.get(k) != row.get(k) for k in QUOTED_FIELDS)
|
| 870 |
|
| 871 |
|
| 872 |
-
def
|
| 873 |
-
with open_mirror() as cur:
|
| 874 |
-
if cur is None:
|
| 875 |
-
return None
|
| 876 |
-
own = our_asins(cur)
|
| 877 |
-
sources = _candidates(cur, sorted(own), newest_week(cur)) if own else []
|
| 878 |
-
stored = _backend(rt).pool_rows(sorted(own))
|
| 879 |
-
values = own | {r[1] for r in sources} | {r['rival_asin'] for r in stored if not r['removed']}
|
| 880 |
-
known = _pool_facts(cur, values, own, candidate_rows=sources)
|
| 881 |
-
missing = sorted(a for a in values if not known.product_type(a))
|
| 882 |
-
prefetched, causes = _read_batches('facts', facts_reader, missing, sleep=sleep, log=log, counts=counts)
|
| 883 |
-
prefetched = {a: prefetched.get(a, {}) for a in missing}
|
| 884 |
-
with open_mirror() as cur:
|
| 885 |
-
if cur is None:
|
| 886 |
-
return None
|
| 887 |
-
found = discover(rt, cur, candidate_rows=sources, prefetched=prefetched)
|
| 888 |
-
return own, found, prefetched, causes
|
| 889 |
-
|
| 890 |
-
|
| 891 |
-
def run_quotes(rt, open_mirror, *, offers, summaries, facts, seller_id=None, run_id=None, now=None,
|
| 892 |
log=None, sleep=time.sleep):
|
| 893 |
-
"""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 894 |
_backend(rt)
|
| 895 |
log = log or (lambda _message: None)
|
| 896 |
now = now or datetime.now(timezone.utc)
|
| 897 |
quote_on = now.date().isoformat()
|
| 898 |
run_id = str(run_id or f'{quote_on}-{uuid.uuid4().hex[:8]}')
|
| 899 |
counts = dict(ours=0, rivals=0, wanted=0, asins=0, priced=0, ranked=0, caused=0, refused=0, throttled=0,
|
| 900 |
-
backoffs=0, discovered=0, week=None
|
| 901 |
start_run(rt, run_id, asin_total=0)
|
| 902 |
log(f'run {run_id} started')
|
| 903 |
done = 0
|
| 904 |
try:
|
| 905 |
try:
|
| 906 |
-
|
| 907 |
-
|
| 908 |
-
|
| 909 |
-
|
|
|
|
| 910 |
except MirrorUnreadable as exc:
|
| 911 |
return _finish(rt, run_id, done, counts, f'The Amazon data could not be read ({exc}).', log)
|
| 912 |
-
counts.update(discovered=found['rivals'], week=found['week']
|
| 913 |
others = sorted(set(rival_asins(rt)) - own)
|
| 914 |
extra = sorted(set(wanted(rt)) - own - set(others))
|
| 915 |
targets = sorted(own) + others + extra
|
|
@@ -918,23 +589,18 @@ def run_quotes(rt, open_mirror, *, offers, summaries, facts, seller_id=None, run
|
|
| 918 |
log(f'run {run_id} quotes {len(targets)} ASINs: {len(own)} ours, {len(others)} rivals, '
|
| 919 |
f'{len(extra)} wanted')
|
| 920 |
before = newest_quotes(rt, targets) if targets else {}
|
| 921 |
-
readers = {'offers': offers, 'summaries': summaries, '
|
| 922 |
-
changed
|
| 923 |
for block in _chunks(targets, PROGRESS_EVERY):
|
| 924 |
-
rows
|
| 925 |
-
prefetched, fact_causes)
|
| 926 |
put_quotes(rt, rows)
|
| 927 |
-
|
| 928 |
-
_tally(counts, rows, own, captured)
|
| 929 |
changed += [row['asin'] for row in rows if _changed(before, row)]
|
| 930 |
done += len(block)
|
| 931 |
mark_run(rt, run_id, progress=done, counts=counts)
|
| 932 |
log(f'run {run_id} progress {done}/{len(targets)}')
|
| 933 |
-
|
| 934 |
-
|
| 935 |
-
if counts['throttled'] or counts['refused'] else None)
|
| 936 |
-
result = _finish(rt, run_id, done, counts, cause, log)
|
| 937 |
-
result.update(changed=tuple(dict.fromkeys(changed)), snapshots=snapshots)
|
| 938 |
return result
|
| 939 |
except Exception as exc:
|
| 940 |
mark_run(rt, run_id, progress=done, finished=True, counts=counts,
|
|
@@ -947,7 +613,7 @@ def _finish(rt, run_id, done, counts, cause, log):
|
|
| 947 |
mark_run(rt, run_id, progress=done, finished=True, counts=counts, cause=cause)
|
| 948 |
log(f'run {run_id} finished: {counts["asins"]} ASINs, {counts["priced"]} priced, {counts["ranked"]} ranked, '
|
| 949 |
f'{counts["caused"]} with a cause' + (f'. {cause}' if cause else ''))
|
| 950 |
-
return {'id': run_id, 'progress': done, 'cause': cause, 'counts': dict(counts), 'changed': ()
|
| 951 |
|
| 952 |
|
| 953 |
# --------------------------------------------------------------------------- competition status and rollups
|
|
@@ -1217,70 +883,6 @@ def validate(rt, tables, plan):
|
|
| 1217 |
'rows': len(parents), 'mismatches': bad['parent']}]
|
| 1218 |
|
| 1219 |
|
| 1220 |
-
# --------------------------------------------------------------------------- pool validation
|
| 1221 |
-
|
| 1222 |
-
def _pool_expected(cur, targets):
|
| 1223 |
-
"""Recompute raw click money independently of discovery's candidate and money queries."""
|
| 1224 |
-
required = {'asin', 'search_query_data_search_query', 'start_date', 'end_date',
|
| 1225 |
-
'purchase_data_total_purchase_count', 'purchase_data_total_median_purchase_price_amount'}
|
| 1226 |
-
clicks = {'search_term', 'clicked_asin', 'click_share', 'click_share_rank', '_period_start', '_period_end'}
|
| 1227 |
-
if not required <= _columns(cur, SQP_TABLE) or not clicks <= _columns(cur, BA_TABLE):
|
| 1228 |
-
return {}
|
| 1229 |
-
rows = _read(cur, f'''
|
| 1230 |
-
WITH monthly AS (
|
| 1231 |
-
SELECT upper(trim(CAST(asin AS VARCHAR))) AS owner,
|
| 1232 |
-
lower(trim(CAST(search_query_data_search_query AS VARCHAR))) AS query,
|
| 1233 |
-
TRY_CAST(end_date AS DATE) AS ends,
|
| 1234 |
-
TRY_CAST(purchase_data_total_purchase_count AS DOUBLE) *
|
| 1235 |
-
TRY_CAST(purchase_data_total_median_purchase_price_amount AS DOUBLE) AS money
|
| 1236 |
-
FROM {SQP_TABLE}
|
| 1237 |
-
WHERE TRY_CAST(end_date AS DATE) > TRY_CAST(start_date AS DATE) + INTERVAL '7 days'
|
| 1238 |
-
AND TRY_CAST(end_date AS DATE) < CURRENT_DATE),
|
| 1239 |
-
newest_month AS (SELECT owner, max(ends) AS ends FROM monthly GROUP BY owner),
|
| 1240 |
-
searches AS (
|
| 1241 |
-
SELECT m.owner, m.query, max(m.money) AS money FROM monthly m
|
| 1242 |
-
JOIN newest_month n ON n.owner = m.owner AND n.ends = m.ends
|
| 1243 |
-
GROUP BY m.owner, m.query HAVING max(m.money) > 0),
|
| 1244 |
-
report AS (
|
| 1245 |
-
SELECT lower(trim(CAST(search_term AS VARCHAR))) AS query,
|
| 1246 |
-
upper(trim(CAST(clicked_asin AS VARCHAR))) AS rival,
|
| 1247 |
-
TRY_CAST(click_share AS DOUBLE) AS share,
|
| 1248 |
-
TRY_CAST(click_share_rank AS INTEGER) AS position, {_week_sql()} AS starts
|
| 1249 |
-
FROM {BA_TABLE} WHERE {weekly_period_sql()}),
|
| 1250 |
-
term_clicks AS (
|
| 1251 |
-
SELECT query, rival, max(share) AS share, min(position) AS position FROM report
|
| 1252 |
-
WHERE starts = (SELECT max(starts) FROM report) AND position BETWEEN 1 AND 3
|
| 1253 |
-
GROUP BY query, rival)
|
| 1254 |
-
SELECT s.owner, c.rival, sum(s.money * coalesce(c.share, 0) * 0.01),
|
| 1255 |
-
sum(CASE WHEN c.position = 1 THEN 1 ELSE 0 END)
|
| 1256 |
-
FROM searches s JOIN term_clicks c ON c.query = s.query
|
| 1257 |
-
WHERE s.owner IN {_in(targets)} GROUP BY s.owner, c.rival''', targets)
|
| 1258 |
-
return {(a, r): (float(m), int(n)) for a, r, m, n in rows}
|
| 1259 |
-
|
| 1260 |
-
|
| 1261 |
-
def validate_pool(cur, rt, targets):
|
| 1262 |
-
"""Tie stored auto-pool money and term wins to an independent raw-source aggregation."""
|
| 1263 |
-
targets = _asins(targets)
|
| 1264 |
-
expected = _pool_expected(cur, targets) if targets else {}
|
| 1265 |
-
bad_money, bad_terms, count = [], [], 0
|
| 1266 |
-
backend = _backend(rt)
|
| 1267 |
-
for target in targets:
|
| 1268 |
-
for row in backend.rivals(target):
|
| 1269 |
-
if row['source'] != 'auto':
|
| 1270 |
-
continue
|
| 1271 |
-
count += 1
|
| 1272 |
-
money, terms = expected.get((target, row['rival_asin']), (0, 0))
|
| 1273 |
-
label = f'{target} {row["rival_asin"]}'
|
| 1274 |
-
if _differs(row['money_won'], money, 0.000001):
|
| 1275 |
-
bad_money.append(label)
|
| 1276 |
-
if row['terms_won'] != terms:
|
| 1277 |
-
bad_terms.append(label)
|
| 1278 |
-
return [{'check': 'pool money won ties to Brand Analytics and search report rows',
|
| 1279 |
-
'ok': not bad_money, 'rows': count, 'mismatches': bad_money},
|
| 1280 |
-
{'check': 'pool terms won ties to Brand Analytics and search report rows',
|
| 1281 |
-
'ok': not bad_terms, 'rows': count, 'mismatches': bad_terms}]
|
| 1282 |
-
|
| 1283 |
-
|
| 1284 |
# --------------------------------------------------------------------------- storage
|
| 1285 |
|
| 1286 |
def _ddl(table):
|
|
@@ -1351,10 +953,6 @@ class PgRivalStore:
|
|
| 1351 |
"""Create every rival table and index, only on a write."""
|
| 1352 |
for name in TABLES:
|
| 1353 |
con.execute(_ddl(self.t(name))[name])
|
| 1354 |
-
for name, fields in POOL_FIELDS.items():
|
| 1355 |
-
for column, (kind, default) in fields.items():
|
| 1356 |
-
con.execute(f'ALTER TABLE {self.t(name)} ADD COLUMN IF NOT EXISTS {column} {kind} '
|
| 1357 |
-
f'DEFAULT {default}')
|
| 1358 |
con.execute(f'CREATE INDEX IF NOT EXISTS "amazon_rivals_rival" ON {self.t(RIVALS)} (tenant, rival_asin)')
|
| 1359 |
|
| 1360 |
@contextmanager
|
|
@@ -1364,49 +962,29 @@ class PgRivalStore:
|
|
| 1364 |
self.ensure_schema(con)
|
| 1365 |
yield con
|
| 1366 |
|
| 1367 |
-
def _select(self, con, name, columns, where, args, order=''
|
| 1368 |
if not self._exists(con, name):
|
| 1369 |
return []
|
| 1370 |
-
|
| 1371 |
-
fields = POOL_FIELDS.get(name, {})
|
| 1372 |
-
if fields.keys() & set(columns) or name == RIVALS:
|
| 1373 |
-
existing = {r[0] for r in con.execute(
|
| 1374 |
-
'SELECT column_name FROM information_schema.columns WHERE table_schema = %s AND table_name = %s',
|
| 1375 |
-
(self.schema, name)).fetchall()}
|
| 1376 |
-
projection = [f'{fields[c][1]} AS {c}' if c in fields and c not in existing else c for c in columns]
|
| 1377 |
-
if name == RIVALS and 'eligible' in existing and not include_ineligible:
|
| 1378 |
-
where = f'({where}) AND eligible = TRUE' if where else 'eligible = TRUE'
|
| 1379 |
-
rows = con.execute(f'SELECT {", ".join(projection)} FROM {self.t(name)} WHERE tenant = %s'
|
| 1380 |
f'{" AND " + where if where else ""}{order}',
|
| 1381 |
(self.tenant_slug, *args)).fetchall()
|
| 1382 |
return [dict(zip(columns, row)) for row in rows]
|
| 1383 |
|
| 1384 |
@staticmethod
|
| 1385 |
def _rival(row):
|
| 1386 |
-
return {**row, 'removed': bool(row['removed']), '
|
| 1387 |
-
'origin': 'added' if row['source'] == 'added' else row['origin'],
|
| 1388 |
'click_share': None if row['click_share'] is None else float(row['click_share'])}
|
| 1389 |
|
| 1390 |
-
def
|
| 1391 |
-
"""
|
|
|
|
| 1392 |
with self._connection() as con:
|
| 1393 |
-
|
| 1394 |
-
con, RIVALS,
|
| 1395 |
-
|
| 1396 |
-
|
| 1397 |
-
|
| 1398 |
-
|
| 1399 |
-
for row in rows:
|
| 1400 |
-
if row['removed']:
|
| 1401 |
-
continue
|
| 1402 |
-
target, rival = row['asin'], row['rival_asin']
|
| 1403 |
-
allowed = (facts.eligible(target, rival) if facts.product_type(target) else
|
| 1404 |
-
row['eligible'] and rival not in facts.own_asins
|
| 1405 |
-
and _other_brand(facts.latest.get(rival, {}).get('brand'), facts.own_brands))
|
| 1406 |
-
if allowed != row['eligible']:
|
| 1407 |
-
con.execute(f'UPDATE {self.t(RIVALS)} SET eligible = %s '
|
| 1408 |
-
'WHERE tenant = %s AND asin = %s AND rival_asin = %s',
|
| 1409 |
-
(allowed, self.tenant_slug, target, rival))
|
| 1410 |
|
| 1411 |
def replace_auto(self, targets, rows, discovery):
|
| 1412 |
with self._write() as con:
|
|
@@ -1425,9 +1003,7 @@ class PgRivalStore:
|
|
| 1425 |
+ ', '.join(f'({_marks(len(cols))})' for _ in part)
|
| 1426 |
+ ' ON CONFLICT (tenant, asin) DO UPDATE SET week = excluded.week, '
|
| 1427 |
'discovered_at = excluded.discovered_at, cause = excluded.cause, '
|
| 1428 |
-
'auto_count = excluded.auto_count
|
| 1429 |
-
'keyword_ranks_on = excluded.keyword_ranks_on, '
|
| 1430 |
-
'product_type = excluded.product_type, held = excluded.held',
|
| 1431 |
tuple(v for row in part for v in (self.tenant_slug, *(row[c] for c in DISCOVERY_COLUMNS))))
|
| 1432 |
|
| 1433 |
def discovery(self, targets):
|
|
@@ -1435,41 +1011,14 @@ class PgRivalStore:
|
|
| 1435 |
rows = []
|
| 1436 |
for part in _chunks(list(targets)):
|
| 1437 |
rows += self._select(con, DISCOVERY, DISCOVERY_COLUMNS, f'asin IN ({_marks(len(part))})', part)
|
| 1438 |
-
return {row['asin']: {**row,
|
| 1439 |
-
('week', 'month', 'keyword_ranks_on')}} for row in rows}
|
| 1440 |
|
| 1441 |
def rivals(self, target):
|
| 1442 |
with self._connection() as con:
|
| 1443 |
-
rows = self._select(con, RIVALS, RIVAL_COLUMNS, 'asin = %s AND removed = %s', (target, False)
|
| 1444 |
-
|
| 1445 |
-
|
| 1446 |
-
|
| 1447 |
-
return sorted(enriched, key=lambda row: (row['source'] != 'added', *_pool_order(row)))
|
| 1448 |
-
|
| 1449 |
-
def rank_averages(self, values):
|
| 1450 |
-
out = {}
|
| 1451 |
-
with self._connection() as con:
|
| 1452 |
-
if not self._exists(con, QUOTES):
|
| 1453 |
-
return out
|
| 1454 |
-
for part in _chunks(values):
|
| 1455 |
-
rows = con.execute(f'''
|
| 1456 |
-
WITH newest AS (
|
| 1457 |
-
SELECT asin, max(quote_on) AS newest FROM {self.t(QUOTES)}
|
| 1458 |
-
WHERE tenant = %s AND asin IN ({_marks(len(part))}) GROUP BY asin),
|
| 1459 |
-
groups AS (
|
| 1460 |
-
SELECT n.asin, n.newest, q.display_group FROM newest n
|
| 1461 |
-
JOIN {self.t(QUOTES)} q ON q.asin = n.asin AND q.quote_on = n.newest
|
| 1462 |
-
WHERE q.tenant = %s)
|
| 1463 |
-
SELECT q.asin, CASE WHEN count(q.sales_rank) >= 7 THEN avg(q.sales_rank) END,
|
| 1464 |
-
count(q.sales_rank)
|
| 1465 |
-
FROM groups m JOIN {self.t(QUOTES)} q ON q.asin = m.asin
|
| 1466 |
-
AND q.display_group = m.display_group AND q.quote_on >= m.newest - 29
|
| 1467 |
-
AND q.quote_on <= m.newest
|
| 1468 |
-
WHERE q.tenant = %s GROUP BY q.asin''',
|
| 1469 |
-
(self.tenant_slug, *part, self.tenant_slug, self.tenant_slug)).fetchall()
|
| 1470 |
-
out.update({a: dict(rank_average=None if avg is None else float(avg), rank_days=n)
|
| 1471 |
-
for a, avg, n in rows})
|
| 1472 |
-
return out
|
| 1473 |
|
| 1474 |
def removed_count(self, target):
|
| 1475 |
with self._connection() as con:
|
|
@@ -1483,15 +1032,11 @@ class PgRivalStore:
|
|
| 1483 |
with self._write() as con:
|
| 1484 |
got = con.execute(f'INSERT INTO {self.t(RIVALS)} ({", ".join(cols)}) VALUES ({_marks(len(cols))}) '
|
| 1485 |
"ON CONFLICT (tenant, asin, rival_asin) DO UPDATE SET source = 'added', "
|
| 1486 |
-
"origin = 'added', money_won = 0, terms_won = 0, term_rank = NULL, "
|
| 1487 |
'removed = excluded.removed, added_by = excluded.added_by, '
|
| 1488 |
'updated_at = excluded.updated_at, '
|
| 1489 |
'title = coalesce(excluded.title, title) '
|
| 1490 |
f'RETURNING {", ".join(RIVAL_COLUMNS)}',
|
| 1491 |
(self.tenant_slug, *(row[c] for c in RIVAL_COLUMNS))).fetchone()
|
| 1492 |
-
if not dict(zip(RIVAL_COLUMNS, got))['eligible']:
|
| 1493 |
-
con.execute(f'UPDATE {self.t(DISCOVERY)} SET week = NULL, month = NULL, keyword_ranks_on = NULL '
|
| 1494 |
-
'WHERE tenant = %s AND asin = %s', (self.tenant_slug, row['asin']))
|
| 1495 |
return self._rival(dict(zip(RIVAL_COLUMNS, got)))
|
| 1496 |
|
| 1497 |
def remove(self, target, rival, user, stamp):
|
|
@@ -1505,13 +1050,15 @@ class PgRivalStore:
|
|
| 1505 |
return self._rival(dict(zip(RIVAL_COLUMNS, got)))
|
| 1506 |
|
| 1507 |
def rivals_of(self, targets):
|
| 1508 |
-
"""`{asin: [rival ASIN]}` of the rivals not removed."""
|
| 1509 |
out = {}
|
| 1510 |
with self._connection() as con:
|
| 1511 |
for part in _chunks(list(targets)):
|
| 1512 |
for row in self._select(con, RIVALS, ('asin', 'rival_asin'),
|
| 1513 |
f'asin IN ({_marks(len(part))}) AND removed = %s', (*part, False),
|
| 1514 |
-
|
|
|
|
|
|
|
| 1515 |
out.setdefault(row['asin'], []).append(row['rival_asin'])
|
| 1516 |
return out
|
| 1517 |
|
|
@@ -1532,10 +1079,6 @@ class PgRivalStore:
|
|
| 1532 |
with self._connection() as con:
|
| 1533 |
if not (self._exists(con, QUOTES) and self._exists(con, RIVALS)):
|
| 1534 |
return own, rows
|
| 1535 |
-
columns = {r[0] for r in con.execute(
|
| 1536 |
-
'SELECT column_name FROM information_schema.columns WHERE table_schema = %s AND table_name = %s',
|
| 1537 |
-
(self.schema, RIVALS)).fetchall()}
|
| 1538 |
-
eligibility = 'AND r.eligible = TRUE ' if 'eligible' in columns else ''
|
| 1539 |
for part in _chunks(list(targets)):
|
| 1540 |
marks = _marks(len(part))
|
| 1541 |
for target, rank, group in con.execute(
|
|
@@ -1546,8 +1089,7 @@ class PgRivalStore:
|
|
| 1546 |
rows += [tuple(r) for r in con.execute(
|
| 1547 |
f'SELECT r.asin, q.{RIVAL_PRICE}, q.sales_rank, q.display_group FROM {self.t(RIVALS)} AS r '
|
| 1548 |
f'JOIN {self.t(QUOTES)} AS q ON q.tenant = r.tenant AND q.asin = r.rival_asin '
|
| 1549 |
-
f'WHERE r.tenant = %s AND r.removed = %s {
|
| 1550 |
-
f'AND r.asin IN ({marks}) AND {newest}',
|
| 1551 |
(self.tenant_slug, False, *part)).fetchall()]
|
| 1552 |
return own, rows
|
| 1553 |
|
|
|
|
| 1 |
+
"""Per-tenant PostgreSQL rivals of our ASINs, found from Brand Analytics, plus their daily quotes.
|
| 2 |
|
| 3 |
Discovery reads the Amazon mirror through a cursor the caller opens: core imports no harness
|
| 4 |
module (verify_layering) and no App module (rule 13). Tables are created on the first write.
|
| 5 |
"""
|
| 6 |
|
| 7 |
from contextlib import contextmanager
|
|
|
|
| 8 |
from datetime import date, datetime, timedelta, timezone
|
| 9 |
import json
|
| 10 |
import re
|
|
|
|
| 19 |
WANTED = 'amazon_rival_wanted'
|
| 20 |
TABLES = (RIVALS, DISCOVERY, QUOTES, RUNS, WANTED)
|
| 21 |
|
| 22 |
+
#: R23: at most this many auto-found rivals per ASIN, by click share.
|
| 23 |
+
AUTO_LIMIT = 10
|
| 24 |
#: Brand Analytics keeps the top 3 clicked ASINs of each search term.
|
| 25 |
TOP_CLICKED = 3
|
| 26 |
SOURCES = frozenset(('auto', 'added'))
|
|
|
|
| 29 |
BA_TABLE = 'amzn_brand_analytics'
|
| 30 |
LISTINGS_TABLE = 'amzn_listings'
|
| 31 |
PRICING_TABLE = 'amzn_pricing'
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 32 |
|
| 33 |
NO_TOP3 = "Not in any search term's top 3 clicked"
|
| 34 |
NO_WEEK = 'Brand Analytics has no dated weekly reports'
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 35 |
NOT_OURS = 'Not an ASIN in your Amazon listings'
|
| 36 |
PARENT_ROLLS_UP = 'A Parent ASIN has no rival list. Its fields roll up its child ASINs.'
|
|
|
|
| 37 |
ASIN_RE = re.compile(r'[A-Z0-9]{10}')
|
| 38 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 39 |
RIVAL_COLUMNS = ('asin', 'rival_asin', 'source', 'removed', 'click_share', 'term', 'title', 'week',
|
| 40 |
+
'added_by', 'updated_at')
|
| 41 |
+
DISCOVERY_COLUMNS = ('asin', 'week', 'discovered_at', 'cause', 'auto_count')
|
| 42 |
QUOTE_COLUMNS = ('asin', 'quote_on', 'offer_price', 'buy_box_price', 'sales_rank', 'display_group',
|
| 43 |
'offer_cause', 'buy_box_cause', 'rank_cause', 'offers', 'currency', 'run_id', 'quoted_at')
|
| 44 |
RUN_COLUMNS = ('id', 'started_at', 'progress', 'progress_at', 'finished_at', 'asin_total', 'priced',
|
|
|
|
| 175 |
return [r[0] for r in rows if r[0]]
|
| 176 |
|
| 177 |
|
| 178 |
+
def _candidates(cur, targets, week, per_asin):
|
| 179 |
+
"""Ranked co-clicked ASINs per target, bounded in SQL to `per_asin` rows each."""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 180 |
cols = _columns(cur, BA_TABLE)
|
| 181 |
+
share = 'TRY_CAST("click_share" AS DOUBLE)' if 'click_share' in cols else 'CAST(NULL AS DOUBLE)'
|
| 182 |
+
title = 'CAST("clicked_item_name" AS VARCHAR)' if 'clicked_item_name' in cols else 'CAST(NULL AS VARCHAR)'
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 183 |
sql = f'''
|
| 184 |
+
WITH ours AS (
|
| 185 |
+
SELECT DISTINCT upper(trim(CAST("asin1" AS VARCHAR))) AS asin FROM {LISTINGS_TABLE}
|
| 186 |
+
WHERE "asin1" IS NOT NULL),
|
| 187 |
+
week AS (
|
| 188 |
+
SELECT "search_term" AS term, upper(trim(CAST("clicked_asin" AS VARCHAR))) AS clicked,
|
| 189 |
+
{share} AS share, {title} AS title
|
| 190 |
+
FROM {BA_TABLE}
|
| 191 |
+
WHERE TRY_CAST("click_share_rank" AS INTEGER) BETWEEN 1 AND {TOP_CLICKED}
|
| 192 |
+
AND ({weekly_period_sql()}) AND {_week_sql()} = CAST(? AS DATE)),
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 193 |
hits AS (
|
| 194 |
+
SELECT DISTINCT term, clicked AS asin FROM week WHERE clicked IN {_in(targets)}),
|
| 195 |
+
cands AS (
|
| 196 |
+
SELECT h.asin, w.clicked AS rival, w.share, w.term, w.title
|
| 197 |
+
FROM hits h JOIN week w ON w.term = h.term
|
| 198 |
+
WHERE w.clicked <> '' AND w.clicked NOT IN (SELECT asin FROM ours)),
|
| 199 |
+
best AS (
|
| 200 |
+
SELECT asin, rival, max(share) AS share,
|
| 201 |
+
arg_max(term, coalesce(share, -1)) AS term, arg_max(title, coalesce(share, -1)) AS title
|
| 202 |
+
FROM cands GROUP BY asin, rival),
|
| 203 |
+
ranked AS (
|
| 204 |
+
SELECT *, row_number() OVER (PARTITION BY asin ORDER BY share DESC NULLS LAST, rival) AS rn
|
| 205 |
+
FROM best)
|
| 206 |
+
SELECT asin, rival, share, term, title FROM ranked WHERE rn <= ? ORDER BY asin, rn'''
|
| 207 |
+
return _read(cur, sql, (week, *targets, per_asin))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 208 |
|
| 209 |
|
| 210 |
# --------------------------------------------------------------------------- discovery
|
| 211 |
|
| 212 |
+
def discover(rt, cur, asins=None):
|
| 213 |
+
"""Rewrite the auto rivals of our ASINs (or of `asins`) from the newest dated week.
|
| 214 |
+
|
| 215 |
+
A removed rival stays removed and a hand-added one stays added. Returns counts and `rowsRead`."""
|
| 216 |
backend = _backend(rt)
|
| 217 |
requested = None if asins is None else _asins(asins)
|
| 218 |
targets = sorted(our_asins(cur, requested))
|
| 219 |
stamp = _now()
|
| 220 |
discovery = {}
|
| 221 |
for missing in sorted(set(requested or ()) - set(targets)):
|
| 222 |
+
discovery[missing] = dict(asin=missing, week=None, discovered_at=stamp, cause=NOT_OURS, auto_count=0)
|
|
|
|
| 223 |
week = newest_week(cur) if targets else None
|
| 224 |
+
found = {}
|
| 225 |
+
rows_read = 0
|
| 226 |
+
if targets and week is not None:
|
| 227 |
+
kept = backend.kept(targets)
|
| 228 |
+
per_asin = AUTO_LIMIT + max((len(v) for v in kept.values()), default=0)
|
| 229 |
+
rows = _candidates(cur, targets, week, per_asin)
|
| 230 |
+
rows_read = len(rows)
|
| 231 |
+
for target, rival, share, term, title in rows:
|
| 232 |
+
if rival in kept.get(target, ()) or len(found.get(target, ())) >= AUTO_LIMIT:
|
| 233 |
+
continue
|
| 234 |
+
found.setdefault(target, []).append(dict(
|
| 235 |
+
asin=target, rival_asin=rival, source='auto', removed=False,
|
| 236 |
+
click_share=None if share is None else float(share), term=term, title=title,
|
| 237 |
+
week=week, added_by=None, updated_at=stamp))
|
| 238 |
for target in targets:
|
| 239 |
+
count = len(found.get(target, ()))
|
| 240 |
+
cause = NO_WEEK if week is None else (None if count else NO_TOP3)
|
| 241 |
+
discovery[target] = dict(asin=target, week=week, discovered_at=stamp, cause=cause, auto_count=count)
|
| 242 |
+
backend.replace_auto(targets, [row for t in targets for row in found.get(t, ())], list(discovery.values()))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 243 |
return {'asins': len(targets), 'rivals': sum(len(v) for v in found.values()), 'week': week,
|
| 244 |
+
'rowsRead': rows_read, 'causes': {k: v['cause'] for k, v in discovery.items() if v['cause']}}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 245 |
|
| 246 |
|
| 247 |
def stale(rt, cur, target):
|
| 248 |
+
"""True when this ASIN's discovery is missing or predates the newest dated week."""
|
| 249 |
row = _backend(rt).discovery([target]).get(target)
|
| 250 |
if row is None:
|
| 251 |
return True
|
| 252 |
+
newest = newest_week(cur)
|
| 253 |
+
return newest is not None and (row['week'] is None or row['week'] < newest)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 254 |
|
| 255 |
|
| 256 |
# --------------------------------------------------------------------------- doors
|
| 257 |
|
| 258 |
def rivals(rt, target):
|
| 259 |
+
"""One ASIN's rivals: hand-added first, then auto by click share, with its discovery row."""
|
| 260 |
target = asin(target)
|
| 261 |
backend = _backend(rt)
|
| 262 |
rows = backend.rivals(target)
|
| 263 |
found = backend.discovery([target]).get(target)
|
| 264 |
return {'asin': target, 'rivals': rows, 'removed': backend.removed_count(target),
|
| 265 |
'week': found['week'] if found else None,
|
|
|
|
|
|
|
| 266 |
'discoveredAt': found['discovered_at'] if found else None,
|
| 267 |
+
'cause': found['cause'] if found else None}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 268 |
|
| 269 |
|
| 270 |
def add(rt, target, rival, *, user, title=None):
|
|
|
|
| 272 |
target, rival = asin(target), asin(rival, 'rival')
|
| 273 |
if rival == target:
|
| 274 |
raise ValueError('An ASIN cannot be its own rival.')
|
| 275 |
+
return _backend(rt).add(dict(asin=target, rival_asin=rival, source='added', removed=False,
|
| 276 |
click_share=None, term=None, title=title, week=None,
|
| 277 |
+
added_by=str(user or '') or None, updated_at=_now()))
|
|
|
|
| 278 |
|
| 279 |
|
| 280 |
def remove(rt, target, rival, *, user):
|
|
|
|
| 338 |
return _backend(rt).runs(int(limit))
|
| 339 |
|
| 340 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 341 |
# --------------------------------------------------------------------------- the daily quotes run
|
| 342 |
|
| 343 |
#: The run row gets a progress marker after each block of this many ASINs.
|
|
|
|
| 351 |
#: An unfinished run, or a finished run with a cause, is retried after this many hours.
|
| 352 |
RETRY_RUN_HOURS = 3
|
| 353 |
|
| 354 |
+
READS = {'offers': 'offers', 'summaries': 'Buy Box price', 'ranks': 'sales rank'}
|
|
|
|
| 355 |
NO_MIRROR = 'The Amazon data is not on this worker yet, so no ASIN was quoted.'
|
| 356 |
NO_OWN_OFFER = 'Amazon returned no offer of yours on this ASIN.'
|
| 357 |
NO_BUY_BOX = 'Amazon returned no Buy Box offer for this ASIN.'
|
|
|
|
| 449 |
|
| 450 |
|
| 451 |
def _keyed(kind, got):
|
| 452 |
+
if kind == 'ranks':
|
| 453 |
return {str(k).upper(): v for k, v in (got or {}).items()}
|
| 454 |
if kind == 'offers':
|
| 455 |
return {str((((r or {}).get('body') or {}).get('payload') or {}).get('ASIN') or '').upper(): r
|
|
|
|
| 489 |
attempt = 0
|
| 490 |
while True:
|
| 491 |
try:
|
| 492 |
+
answers.update(_keyed(kind, fn(part)))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 493 |
break
|
| 494 |
except Exception as exc: # noqa: BLE001
|
| 495 |
if _throttled(exc) and attempt < THROTTLE_RETRIES:
|
|
|
|
| 505 |
return answers, causes
|
| 506 |
|
| 507 |
|
| 508 |
+
def _quote_block(block, own, readers, seller_id, quote_on, run_id, sleep, log, counts):
|
|
|
|
| 509 |
"""One quote row per ASIN of `block`, each with a value or a cause for every field it owns."""
|
| 510 |
mine = [a for a in block if a in own]
|
| 511 |
others = [a for a in block if a not in own]
|
| 512 |
offered, offer_causes = _read_batches('offers', readers['offers'], mine, sleep=sleep, log=log, counts=counts)
|
| 513 |
summed, sum_causes = _read_batches('summaries', readers['summaries'], others, sleep=sleep, log=log,
|
| 514 |
counts=counts)
|
| 515 |
+
ranked, rank_causes = _read_batches('ranks', readers['ranks'], block, sleep=sleep, log=log, counts=counts)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 516 |
stamp = _now()
|
| 517 |
rows = []
|
| 518 |
for target in block:
|
|
|
|
| 531 |
row.update(_summary_quote(summed[target]))
|
| 532 |
else:
|
| 533 |
row['buy_box_cause'] = NO_PRICE
|
| 534 |
+
rank = ranked.get(target) or {}
|
| 535 |
value = rank.get('rank') if isinstance(rank.get('rank'), int) and not isinstance(rank.get('rank'), bool) else None
|
| 536 |
row.update(sales_rank=value, display_group=rank.get('group') if value is not None else None,
|
| 537 |
+
rank_cause=rank_causes.get(target) or (None if value is not None else NO_RANK))
|
| 538 |
rows.append(row)
|
| 539 |
+
return rows
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 540 |
|
| 541 |
|
| 542 |
+
def _tally(counts, rows, own):
|
| 543 |
for row in rows:
|
| 544 |
counts['priced'] += int((row['offer_price'] if row['asin'] in own else None) is not None
|
| 545 |
or row['buy_box_price'] is not None)
|
| 546 |
counts['ranked'] += int(row['sales_rank'] is not None)
|
| 547 |
counts['caused'] += int(bool(row['offer_cause'] or row['buy_box_cause'] or row['rank_cause']))
|
|
|
|
|
|
|
| 548 |
|
| 549 |
|
| 550 |
def _changed(before, row):
|
|
|
|
| 552 |
return old is None or any(old.get(k) != row.get(k) for k in QUOTED_FIELDS)
|
| 553 |
|
| 554 |
|
| 555 |
+
def run_quotes(rt, open_mirror, *, offers, summaries, ranks, seller_id=None, run_id=None, now=None,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 556 |
log=None, sleep=time.sleep):
|
| 557 |
+
"""Rediscover rivals, then quote our ASINs, every rival and every wanted ASIN for today.
|
| 558 |
+
|
| 559 |
+
`open_mirror()` is a context manager yielding a mirror cursor or None. It closes before the first
|
| 560 |
+
SP-API read. `offers`, `summaries` and `ranks` each take a list of at most 20 ASINs and return
|
| 561 |
+
`item_offers` responses, `competitive_summaries` responses and `{asin: {rank, group}}`. A raised
|
| 562 |
+
read is a cause on each ASIN of its batch, and a 429 backs off first. Returns the run's counts
|
| 563 |
+
and `changed`, the ASINs whose quote differs from their previous newest quote."""
|
| 564 |
_backend(rt)
|
| 565 |
log = log or (lambda _message: None)
|
| 566 |
now = now or datetime.now(timezone.utc)
|
| 567 |
quote_on = now.date().isoformat()
|
| 568 |
run_id = str(run_id or f'{quote_on}-{uuid.uuid4().hex[:8]}')
|
| 569 |
counts = dict(ours=0, rivals=0, wanted=0, asins=0, priced=0, ranked=0, caused=0, refused=0, throttled=0,
|
| 570 |
+
backoffs=0, discovered=0, week=None)
|
| 571 |
start_run(rt, run_id, asin_total=0)
|
| 572 |
log(f'run {run_id} started')
|
| 573 |
done = 0
|
| 574 |
try:
|
| 575 |
try:
|
| 576 |
+
with open_mirror() as cur:
|
| 577 |
+
if cur is None:
|
| 578 |
+
return _finish(rt, run_id, done, counts, NO_MIRROR, log)
|
| 579 |
+
found = discover(rt, cur)
|
| 580 |
+
own = our_asins(cur)
|
| 581 |
except MirrorUnreadable as exc:
|
| 582 |
return _finish(rt, run_id, done, counts, f'The Amazon data could not be read ({exc}).', log)
|
| 583 |
+
counts.update(discovered=found['rivals'], week=found['week'])
|
| 584 |
others = sorted(set(rival_asins(rt)) - own)
|
| 585 |
extra = sorted(set(wanted(rt)) - own - set(others))
|
| 586 |
targets = sorted(own) + others + extra
|
|
|
|
| 589 |
log(f'run {run_id} quotes {len(targets)} ASINs: {len(own)} ours, {len(others)} rivals, '
|
| 590 |
f'{len(extra)} wanted')
|
| 591 |
before = newest_quotes(rt, targets) if targets else {}
|
| 592 |
+
readers = {'offers': offers, 'summaries': summaries, 'ranks': ranks}
|
| 593 |
+
changed = []
|
| 594 |
for block in _chunks(targets, PROGRESS_EVERY):
|
| 595 |
+
rows = _quote_block(block, own, readers, seller_id, quote_on, run_id, sleep, log, counts)
|
|
|
|
| 596 |
put_quotes(rt, rows)
|
| 597 |
+
_tally(counts, rows, own)
|
|
|
|
| 598 |
changed += [row['asin'] for row in rows if _changed(before, row)]
|
| 599 |
done += len(block)
|
| 600 |
mark_run(rt, run_id, progress=done, counts=counts)
|
| 601 |
log(f'run {run_id} progress {done}/{len(targets)}')
|
| 602 |
+
result = _finish(rt, run_id, done, counts, None, log)
|
| 603 |
+
result['changed'] = tuple(changed)
|
|
|
|
|
|
|
|
|
|
| 604 |
return result
|
| 605 |
except Exception as exc:
|
| 606 |
mark_run(rt, run_id, progress=done, finished=True, counts=counts,
|
|
|
|
| 613 |
mark_run(rt, run_id, progress=done, finished=True, counts=counts, cause=cause)
|
| 614 |
log(f'run {run_id} finished: {counts["asins"]} ASINs, {counts["priced"]} priced, {counts["ranked"]} ranked, '
|
| 615 |
f'{counts["caused"]} with a cause' + (f'. {cause}' if cause else ''))
|
| 616 |
+
return {'id': run_id, 'progress': done, 'cause': cause, 'counts': dict(counts), 'changed': ()}
|
| 617 |
|
| 618 |
|
| 619 |
# --------------------------------------------------------------------------- competition status and rollups
|
|
|
|
| 883 |
'rows': len(parents), 'mismatches': bad['parent']}]
|
| 884 |
|
| 885 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 886 |
# --------------------------------------------------------------------------- storage
|
| 887 |
|
| 888 |
def _ddl(table):
|
|
|
|
| 953 |
"""Create every rival table and index, only on a write."""
|
| 954 |
for name in TABLES:
|
| 955 |
con.execute(_ddl(self.t(name))[name])
|
|
|
|
|
|
|
|
|
|
|
|
|
| 956 |
con.execute(f'CREATE INDEX IF NOT EXISTS "amazon_rivals_rival" ON {self.t(RIVALS)} (tenant, rival_asin)')
|
| 957 |
|
| 958 |
@contextmanager
|
|
|
|
| 962 |
self.ensure_schema(con)
|
| 963 |
yield con
|
| 964 |
|
| 965 |
+
def _select(self, con, name, columns, where, args, order=''):
|
| 966 |
if not self._exists(con, name):
|
| 967 |
return []
|
| 968 |
+
rows = con.execute(f'SELECT {", ".join(columns)} FROM {self.t(name)} WHERE tenant = %s'
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 969 |
f'{" AND " + where if where else ""}{order}',
|
| 970 |
(self.tenant_slug, *args)).fetchall()
|
| 971 |
return [dict(zip(columns, row)) for row in rows]
|
| 972 |
|
| 973 |
@staticmethod
|
| 974 |
def _rival(row):
|
| 975 |
+
return {**row, 'removed': bool(row['removed']), 'week': _day(row['week']),
|
|
|
|
| 976 |
'click_share': None if row['click_share'] is None else float(row['click_share'])}
|
| 977 |
|
| 978 |
+
def kept(self, targets):
|
| 979 |
+
"""`{asin: {rival}}` of removed and hand-added rivals, which discovery must not touch."""
|
| 980 |
+
out = {}
|
| 981 |
with self._connection() as con:
|
| 982 |
+
for part in _chunks(targets):
|
| 983 |
+
for row in self._select(con, RIVALS, ('asin', 'rival_asin'),
|
| 984 |
+
f"asin IN ({_marks(len(part))}) AND (removed = %s OR source = 'added')",
|
| 985 |
+
(*part, True)):
|
| 986 |
+
out.setdefault(row['asin'], set()).add(row['rival_asin'])
|
| 987 |
+
return out
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 988 |
|
| 989 |
def replace_auto(self, targets, rows, discovery):
|
| 990 |
with self._write() as con:
|
|
|
|
| 1003 |
+ ', '.join(f'({_marks(len(cols))})' for _ in part)
|
| 1004 |
+ ' ON CONFLICT (tenant, asin) DO UPDATE SET week = excluded.week, '
|
| 1005 |
'discovered_at = excluded.discovered_at, cause = excluded.cause, '
|
| 1006 |
+
'auto_count = excluded.auto_count',
|
|
|
|
|
|
|
| 1007 |
tuple(v for row in part for v in (self.tenant_slug, *(row[c] for c in DISCOVERY_COLUMNS))))
|
| 1008 |
|
| 1009 |
def discovery(self, targets):
|
|
|
|
| 1011 |
rows = []
|
| 1012 |
for part in _chunks(list(targets)):
|
| 1013 |
rows += self._select(con, DISCOVERY, DISCOVERY_COLUMNS, f'asin IN ({_marks(len(part))})', part)
|
| 1014 |
+
return {row['asin']: {**row, 'week': _day(row['week'])} for row in rows}
|
|
|
|
| 1015 |
|
| 1016 |
def rivals(self, target):
|
| 1017 |
with self._connection() as con:
|
| 1018 |
+
rows = self._select(con, RIVALS, RIVAL_COLUMNS, 'asin = %s AND removed = %s', (target, False),
|
| 1019 |
+
" ORDER BY CASE WHEN source = 'added' THEN 0 ELSE 1 END, "
|
| 1020 |
+
'CASE WHEN click_share IS NULL THEN 1 ELSE 0 END, click_share DESC, rival_asin')
|
| 1021 |
+
return [self._rival(row) for row in rows]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1022 |
|
| 1023 |
def removed_count(self, target):
|
| 1024 |
with self._connection() as con:
|
|
|
|
| 1032 |
with self._write() as con:
|
| 1033 |
got = con.execute(f'INSERT INTO {self.t(RIVALS)} ({", ".join(cols)}) VALUES ({_marks(len(cols))}) '
|
| 1034 |
"ON CONFLICT (tenant, asin, rival_asin) DO UPDATE SET source = 'added', "
|
|
|
|
| 1035 |
'removed = excluded.removed, added_by = excluded.added_by, '
|
| 1036 |
'updated_at = excluded.updated_at, '
|
| 1037 |
'title = coalesce(excluded.title, title) '
|
| 1038 |
f'RETURNING {", ".join(RIVAL_COLUMNS)}',
|
| 1039 |
(self.tenant_slug, *(row[c] for c in RIVAL_COLUMNS))).fetchone()
|
|
|
|
|
|
|
|
|
|
| 1040 |
return self._rival(dict(zip(RIVAL_COLUMNS, got)))
|
| 1041 |
|
| 1042 |
def remove(self, target, rival, user, stamp):
|
|
|
|
| 1050 |
return self._rival(dict(zip(RIVAL_COLUMNS, got)))
|
| 1051 |
|
| 1052 |
def rivals_of(self, targets):
|
| 1053 |
+
"""`{asin: [rival ASIN]}` of the rivals not removed, hand-added first then by click share."""
|
| 1054 |
out = {}
|
| 1055 |
with self._connection() as con:
|
| 1056 |
for part in _chunks(list(targets)):
|
| 1057 |
for row in self._select(con, RIVALS, ('asin', 'rival_asin'),
|
| 1058 |
f'asin IN ({_marks(len(part))}) AND removed = %s', (*part, False),
|
| 1059 |
+
" ORDER BY asin, CASE WHEN source = 'added' THEN 0 ELSE 1 END, "
|
| 1060 |
+
'CASE WHEN click_share IS NULL THEN 1 ELSE 0 END, click_share DESC, '
|
| 1061 |
+
'rival_asin'):
|
| 1062 |
out.setdefault(row['asin'], []).append(row['rival_asin'])
|
| 1063 |
return out
|
| 1064 |
|
|
|
|
| 1079 |
with self._connection() as con:
|
| 1080 |
if not (self._exists(con, QUOTES) and self._exists(con, RIVALS)):
|
| 1081 |
return own, rows
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1082 |
for part in _chunks(list(targets)):
|
| 1083 |
marks = _marks(len(part))
|
| 1084 |
for target, rank, group in con.execute(
|
|
|
|
| 1089 |
rows += [tuple(r) for r in con.execute(
|
| 1090 |
f'SELECT r.asin, q.{RIVAL_PRICE}, q.sales_rank, q.display_group FROM {self.t(RIVALS)} AS r '
|
| 1091 |
f'JOIN {self.t(QUOTES)} AS q ON q.tenant = r.tenant AND q.asin = r.rival_asin '
|
| 1092 |
+
f'WHERE r.tenant = %s AND r.removed = %s AND r.asin IN ({marks}) AND {newest}',
|
|
|
|
| 1093 |
(self.tenant_slug, False, *part)).fetchall()]
|
| 1094 |
return own, rows
|
| 1095 |
|
platform/core/connector_spec.py
CHANGED
|
@@ -843,16 +843,6 @@ AMAZON_TABLES = (
|
|
| 843 |
identity=("rank_id",),
|
| 844 |
links=(("keyword_id", _AMZN + "sku_keywords", "keyword_id", "many_to_one"),),
|
| 845 |
),
|
| 846 |
-
ConnectorTableSpec(
|
| 847 |
-
key=_AMZN + "competitor_snapshots", native="competitor_run:amazon_quotes_loop.run_tenant",
|
| 848 |
-
source="amzn_competitor_snapshots", label="Amazon Marketing: Competitor snapshots", pinned="asin",
|
| 849 |
-
identity=("asin", "snapshot_date"),
|
| 850 |
-
),
|
| 851 |
-
ConnectorTableSpec(
|
| 852 |
-
key=_AMZN + "competitor_images", native="competitor_run:amazon_quotes_loop.run_tenant",
|
| 853 |
-
source="amzn_competitor_images", label="Amazon Marketing: Competitor images", pinned="asin",
|
| 854 |
-
identity=("asin", "variant", "link"),
|
| 855 |
-
),
|
| 856 |
)
|
| 857 |
|
| 858 |
#: Snapshot reports that keep change history in `amzn_<t>_history`. Equals the REPORT_TYPES keys
|
|
@@ -883,38 +873,6 @@ AMAZON_TABLES = AMAZON_TABLES + tuple(
|
|
| 883 |
|
| 884 |
# Local projections supplement measured vendor columns without changing the table contract.
|
| 885 |
AMAZON_FIELDS = {
|
| 886 |
-
_AMZN + "competitor_snapshots": tuple(
|
| 887 |
-
{"name": name, "type": kind, "label": label, "source": "competitor_run." + name}
|
| 888 |
-
for name, kind, label in (
|
| 889 |
-
("asin", "VARCHAR", "ASIN"),
|
| 890 |
-
("snapshot_date", "DATE", "Snapshot date"),
|
| 891 |
-
("product_type", "VARCHAR", "Product type"),
|
| 892 |
-
("brand", "VARCHAR", "Brand"),
|
| 893 |
-
("title", "VARCHAR", "Title"),
|
| 894 |
-
("bullets", "VARCHAR", "Bullet points"),
|
| 895 |
-
("image_variants", "VARCHAR", "Image variants"),
|
| 896 |
-
("price", "DOUBLE", "Price"),
|
| 897 |
-
("currency", "VARCHAR", "Currency"),
|
| 898 |
-
("prime", "BOOLEAN", "Prime"),
|
| 899 |
-
("offers", "BIGINT", "Offers"),
|
| 900 |
-
("sales_rank", "BIGINT", "Sales rank"),
|
| 901 |
-
("display_group", "VARCHAR", "Display group"),
|
| 902 |
-
("rank_cat", "VARCHAR", "Rank category"),
|
| 903 |
-
("image", "VARCHAR", "Image"),
|
| 904 |
-
)
|
| 905 |
-
),
|
| 906 |
-
_AMZN + "competitor_images": tuple(
|
| 907 |
-
{"name": name, "type": kind, "label": label, "source": "competitor_run." + name}
|
| 908 |
-
for name, kind, label in (
|
| 909 |
-
("asin", "VARCHAR", "ASIN"),
|
| 910 |
-
("variant", "VARCHAR", "Variant"),
|
| 911 |
-
("link", "VARCHAR", "Link"),
|
| 912 |
-
("sha256", "VARCHAR", "SHA256"),
|
| 913 |
-
("width", "BIGINT", "Width"),
|
| 914 |
-
("height", "BIGINT", "Height"),
|
| 915 |
-
("snapshot_date", "DATE", "Snapshot date"),
|
| 916 |
-
)
|
| 917 |
-
),
|
| 918 |
_AMZN + "keyword_ranks": tuple(
|
| 919 |
{"name": name, "type": kind, "label": label, "source": source}
|
| 920 |
for name, kind, label, source in (
|
|
@@ -3637,9 +3595,6 @@ AMAZON_DEFAULT_COLUMNS = {
|
|
| 3637 |
"cart_adds", "purchases", "backend_keywords", "source_basis"),
|
| 3638 |
_AMZN + "keyword_ranks": ("keyword", "date", "organic_rank", "sponsored_rank",
|
| 3639 |
"sponsored_rank_basis", "sku", "asin"),
|
| 3640 |
-
_AMZN + "competitor_snapshots": ("asin", "snapshot_date", "brand", "title", "product_type",
|
| 3641 |
-
"price", "currency", "prime", "offers", "sales_rank", "rank_cat"),
|
| 3642 |
-
_AMZN + "competitor_images": ("asin", "variant", "link", "sha256", "width", "height", "snapshot_date"),
|
| 3643 |
}
|
| 3644 |
# A history table opens on its source's columns plus the validity dates. An undeclared source stays undeclared.
|
| 3645 |
AMAZON_DEFAULT_COLUMNS.update({_AMZN + _t + "_history": (*AMAZON_DEFAULT_COLUMNS[_AMZN + _t], "valid_from", "valid_to")
|
|
@@ -3767,7 +3722,7 @@ AMAZON_GROUPS = {
|
|
| 3767 |
**{_AMZN + t: "Advertising" for t in (
|
| 3768 |
"brand_analytics", "search_query_performance", "search_catalog_performance",
|
| 3769 |
"market_basket", "repeat_purchase", "promotion_performance", "coupon_performance",
|
| 3770 |
-
"sku_keywords", "keyword_ranks"
|
| 3771 |
}
|
| 3772 |
AMAZON_GROUPS.update({_AMZN + _t + "_history": AMAZON_GROUPS[_AMZN + _t] for _t in AMAZON_HISTORY_SOURCES})
|
| 3773 |
|
|
|
|
| 843 |
identity=("rank_id",),
|
| 844 |
links=(("keyword_id", _AMZN + "sku_keywords", "keyword_id", "many_to_one"),),
|
| 845 |
),
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 846 |
)
|
| 847 |
|
| 848 |
#: Snapshot reports that keep change history in `amzn_<t>_history`. Equals the REPORT_TYPES keys
|
|
|
|
| 873 |
|
| 874 |
# Local projections supplement measured vendor columns without changing the table contract.
|
| 875 |
AMAZON_FIELDS = {
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 876 |
_AMZN + "keyword_ranks": tuple(
|
| 877 |
{"name": name, "type": kind, "label": label, "source": source}
|
| 878 |
for name, kind, label, source in (
|
|
|
|
| 3595 |
"cart_adds", "purchases", "backend_keywords", "source_basis"),
|
| 3596 |
_AMZN + "keyword_ranks": ("keyword", "date", "organic_rank", "sponsored_rank",
|
| 3597 |
"sponsored_rank_basis", "sku", "asin"),
|
|
|
|
|
|
|
|
|
|
| 3598 |
}
|
| 3599 |
# A history table opens on its source's columns plus the validity dates. An undeclared source stays undeclared.
|
| 3600 |
AMAZON_DEFAULT_COLUMNS.update({_AMZN + _t + "_history": (*AMAZON_DEFAULT_COLUMNS[_AMZN + _t], "valid_from", "valid_to")
|
|
|
|
| 3722 |
**{_AMZN + t: "Advertising" for t in (
|
| 3723 |
"brand_analytics", "search_query_performance", "search_catalog_performance",
|
| 3724 |
"market_basket", "repeat_purchase", "promotion_performance", "coupon_performance",
|
| 3725 |
+
"sku_keywords", "keyword_ranks")},
|
| 3726 |
}
|
| 3727 |
AMAZON_GROUPS.update({_AMZN + _t + "_history": AMAZON_GROUPS[_AMZN + _t] for _t in AMAZON_HISTORY_SOURCES})
|
| 3728 |
|
platform/core/external_tools.py
CHANGED
|
@@ -267,7 +267,7 @@ _APPLICATION = {
|
|
| 267 |
"templateKind": {"enum": ["route", "catalog", "competitor_watch", "ranking_diagnostic",
|
| 268 |
"amazon_operator", "restock_planner", "reimbursements", "aged_stock",
|
| 269 |
"opportunity_finder", "returns_refunds", "repeat_buyers", "payouts", "account_health",
|
| 270 |
-
"amazon_dashboard", "amazon_profit", "alerts"
|
| 271 |
"fieldMap": {"type": "object", "additionalProperties": _STRING},
|
| 272 |
"navPlacement": _object({"section": {"const": "app"},
|
| 273 |
"order": {"type": "integer", "minimum": 0, "maximum": 999}},
|
|
|
|
| 267 |
"templateKind": {"enum": ["route", "catalog", "competitor_watch", "ranking_diagnostic",
|
| 268 |
"amazon_operator", "restock_planner", "reimbursements", "aged_stock",
|
| 269 |
"opportunity_finder", "returns_refunds", "repeat_buyers", "payouts", "account_health",
|
| 270 |
+
"amazon_dashboard", "amazon_profit", "alerts"]},
|
| 271 |
"fieldMap": {"type": "object", "additionalProperties": _STRING},
|
| 272 |
"navPlacement": _object({"section": {"const": "app"},
|
| 273 |
"order": {"type": "integer", "minimum": 0, "maximum": 999}},
|
platform/core/shares.py
CHANGED
|
@@ -80,7 +80,7 @@ SHARES_KEY = 'object_shares'
|
|
| 80 |
#: here is this tuple member. That is the point of one registry: the new object's WALL is written
|
| 81 |
#: once in `core.perm_scope`, and its DOORS once in `routes_shares.py` — never a fourth
|
| 82 |
#: permission decision in a fourth file ([[one-evaluator-per-question]]).
|
| 83 |
-
KINDS = ('view', 'folder', 'database', 'field', 'cohort', 'workbook'
|
| 84 |
|
| 85 |
#: `*` is "everyone who can already open the surface". It is NOT "every account on the platform".
|
| 86 |
#: Spelled as a single character so it can never collide with a username (usernames are lower-case
|
|
@@ -692,32 +692,13 @@ def reconcile_database_grants(st=None, *, tenant=None, table_keys=None):
|
|
| 692 |
return repaired[0]
|
| 693 |
|
| 694 |
|
| 695 |
-
def _synthetic_qa_publication(target, table_keys):
|
| 696 |
-
"""Identify private scratch publications outside the legacy account plane."""
|
| 697 |
-
from core import user_tables, users
|
| 698 |
-
|
| 699 |
-
qa = users.qa_identity(users._QA_USERNAME)
|
| 700 |
-
if (not qa or not table_keys or getattr(target, 'key', None) != qa['tenant']
|
| 701 |
-
or getattr(getattr(target, 'tenant', None), 'key', None) != qa['tenant']):
|
| 702 |
-
return False
|
| 703 |
-
for key in table_keys:
|
| 704 |
-
if not isinstance(key, str) or not key.startswith(user_tables.KEY_PREFIX):
|
| 705 |
-
return False
|
| 706 |
-
definition = user_tables.get_definition(key, st=target)
|
| 707 |
-
if (not isinstance(definition, dict) or definition.get('key') != key
|
| 708 |
-
or definition.get('createdBy') != qa['username']
|
| 709 |
-
or definition.get('source') != user_tables.BLANK_SOURCE):
|
| 710 |
-
return False
|
| 711 |
-
return True
|
| 712 |
-
|
| 713 |
-
|
| 714 |
def _strict_accounts():
|
| 715 |
"""Every account record, read strictly: the account rows on PostgreSQL, else the users document."""
|
| 716 |
from core import account_rows, users
|
| 717 |
# On PostgreSQL the users document is a layout marker; reading it repairs no account at all.
|
| 718 |
if account_rows._pg_source(users.store):
|
| 719 |
return dict(account_rows._all(users.store))
|
| 720 |
-
registry = users.
|
| 721 |
if account_rows.is_marker(registry):
|
| 722 |
raise RuntimeError('the account registry is unavailable')
|
| 723 |
return registry
|
|
@@ -727,9 +708,6 @@ def repair_published_database_grants(st=None, *, table_keys):
|
|
| 727 |
"""Attempt one repair after publication, deferring failures to boot/editor except freeze refusal."""
|
| 728 |
from core import freeze_notice
|
| 729 |
try:
|
| 730 |
-
table_keys = tuple(table_keys)
|
| 731 |
-
if _synthetic_qa_publication(_st(st), table_keys):
|
| 732 |
-
return 0
|
| 733 |
return reconcile_database_grants(st, table_keys=table_keys)
|
| 734 |
except freeze_notice.DataInputFrozen:
|
| 735 |
raise
|
|
|
|
| 80 |
#: here is this tuple member. That is the point of one registry: the new object's WALL is written
|
| 81 |
#: once in `core.perm_scope`, and its DOORS once in `routes_shares.py` — never a fourth
|
| 82 |
#: permission decision in a fourth file ([[one-evaluator-per-question]]).
|
| 83 |
+
KINDS = ('view', 'folder', 'database', 'field', 'cohort', 'workbook')
|
| 84 |
|
| 85 |
#: `*` is "everyone who can already open the surface". It is NOT "every account on the platform".
|
| 86 |
#: Spelled as a single character so it can never collide with a username (usernames are lower-case
|
|
|
|
| 692 |
return repaired[0]
|
| 693 |
|
| 694 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 695 |
def _strict_accounts():
|
| 696 |
"""Every account record, read strictly: the account rows on PostgreSQL, else the users document."""
|
| 697 |
from core import account_rows, users
|
| 698 |
# On PostgreSQL the users document is a layout marker; reading it repairs no account at all.
|
| 699 |
if account_rows._pg_source(users.store):
|
| 700 |
return dict(account_rows._all(users.store))
|
| 701 |
+
registry = users.store._read_strict('users')
|
| 702 |
if account_rows.is_marker(registry):
|
| 703 |
raise RuntimeError('the account registry is unavailable')
|
| 704 |
return registry
|
|
|
|
| 708 |
"""Attempt one repair after publication, deferring failures to boot/editor except freeze refusal."""
|
| 709 |
from core import freeze_notice
|
| 710 |
try:
|
|
|
|
|
|
|
|
|
|
| 711 |
return reconcile_database_grants(st, table_keys=table_keys)
|
| 712 |
except freeze_notice.DataInputFrozen:
|
| 713 |
raise
|
platform/harness/amazon_client.py
CHANGED
|
@@ -450,11 +450,8 @@ REPORT_TYPES = {
|
|
| 450 |
"report": "GET_BRAND_ANALYTICS_SEARCH_QUERY_PERFORMANCE_REPORT",
|
| 451 |
"history": {"from": "2025-04-01",
|
| 452 |
"evidence": "Monthly FATAL, the period may fall outside the available data range, for 2021-09, "
|
| 453 |
-
"2024-10, 2024-11, 2025-01 and 2025-03; 2025-04 onward landed
|
| 454 |
-
|
| 455 |
-
"evidence": "2026-11-14 probe refused 2025-07-06 and served 2025-10-05. "
|
| 456 |
-
"The intervening twelve weeks are unasked. 2025-07-13 is the "
|
| 457 |
-
"first unasked Sunday, not a proved earliest served floor"}}},
|
| 458 |
"role": "Brand Analytics",
|
| 459 |
"validates_against": "Amazon dataByAsin row count and ASIN purchases per batch vs stored rows for the latest month",
|
| 460 |
"json_root": "dataByAsin", "monthly": True,
|
|
|
|
| 450 |
"report": "GET_BRAND_ANALYTICS_SEARCH_QUERY_PERFORMANCE_REPORT",
|
| 451 |
"history": {"from": "2025-04-01",
|
| 452 |
"evidence": "Monthly FATAL, the period may fall outside the available data range, for 2021-09, "
|
| 453 |
+
"2024-10, 2024-11, 2025-01 and 2025-03; 2025-04 onward landed. Weekly FATAL through the "
|
| 454 |
+
"week of 2025-07-06, DONE 2025-10-05 (2026-11-14)"},
|
|
|
|
|
|
|
|
|
|
| 455 |
"role": "Brand Analytics",
|
| 456 |
"validates_against": "Amazon dataByAsin row count and ASIN purchases per batch vs stored rows for the latest month",
|
| 457 |
"json_root": "dataByAsin", "monthly": True,
|
platform/harness/amazon_schema.py
CHANGED
|
@@ -95,8 +95,6 @@ PROVENANCE = {
|
|
| 95 |
"search_query_performance": "measured_at_sync",
|
| 96 |
"sku_keywords": "derived",
|
| 97 |
"keyword_ranks": "apify_dataset",
|
| 98 |
-
"competitor_snapshots": "competitor_run",
|
| 99 |
-
"competitor_images": "competitor_run",
|
| 100 |
"fee_estimates_api": "measured_at_sync",
|
| 101 |
}
|
| 102 |
|
|
@@ -107,13 +105,10 @@ PROVENANCE.update(dict.fromkeys((t + "_history" for t in connector_spec.AMAZON_H
|
|
| 107 |
REVIEW_UNAVAILABLE_NOTE = (
|
| 108 |
"Amazon's API does not supply written reviews, average star rating or review count. "
|
| 109 |
"Read them on the product page or in Seller Central.")
|
| 110 |
-
SQP_LIMIT_NOTE = "Amazon lists at most 100 searches a month for each product."
|
| 111 |
|
| 112 |
#: Human copy for the disclosure screen, per provenance. ⛔ NO EM DASH OR EN DASH: this reaches a
|
| 113 |
#: screen and standing rule 2 governs it.
|
| 114 |
PROVENANCE_COPY = {
|
| 115 |
-
"competitor_run": ("Daily competitor facts saved by the competitor run. Every snapshot day is kept. "
|
| 116 |
-
"Image rows keep each distinct variant and link, dated when first saved."),
|
| 117 |
"apify_dataset": ("Apify Amazon search dataset joined to each SKU's latest matching keyword row on or before the capture date. "
|
| 118 |
"Sponsored rank uses the vendor rank or ordered sponsored placements in the captured dataset. "
|
| 119 |
"Global position is separate. Unmatched observations remain in the raw mirror."),
|
|
@@ -308,8 +303,8 @@ def catalogue():
|
|
| 308 |
descriptor = descriptors[key]
|
| 309 |
label, native = descriptor.label, descriptor.native
|
| 310 |
table[key] = {"label": label, "native": native}
|
| 311 |
-
if provenance in {"derived", "apify_dataset"
|
| 312 |
-
table["columns"] = [dict(field) for field in AMAZON_FIELDS
|
| 313 |
if field["name"] != STORE]
|
| 314 |
table["note"] = PROVENANCE_COPY[provenance]
|
| 315 |
table["local_columns"] = [dict(field) for field in AMAZON_FIELDS.get(key, ())
|
|
@@ -319,7 +314,7 @@ def catalogue():
|
|
| 319 |
|
| 320 |
def columns_for(table):
|
| 321 |
"""The pinned column list for one table, as `[(name, type)]`. Empty when measured at sync."""
|
| 322 |
-
if PROVENANCE.get(str(table)) in {"derived", "apify_dataset"
|
| 323 |
return []
|
| 324 |
spec = (catalogue().get("tables") or {}).get(str(table)) or {}
|
| 325 |
return [(c["name"], c.get("type") or "VARCHAR") for c in (spec.get("columns") or [])]
|
|
@@ -382,17 +377,17 @@ def _table_spec(table, identity, relationships=(), *, grain="entity", line_discr
|
|
| 382 |
fields = AMAZON_FIELDS.get(DATABASE_PREFIX + table, ())
|
| 383 |
spec["columns"].extend(dict(field) for field in fields if field["name"] != STORE)
|
| 384 |
spec["local_columns"] = [dict(field) for field in fields if field["name"] == STORE]
|
| 385 |
-
|
| 386 |
-
"
|
| 387 |
-
|
| 388 |
-
|
| 389 |
-
|
| 390 |
-
|
| 391 |
-
if
|
| 392 |
spec["columns_source"] = {
|
| 393 |
"path": "platform/core/connector_spec.py", "reader": "connector_spec.AMAZON_FIELDS",
|
| 394 |
-
"provenance":
|
| 395 |
-
"sources":
|
| 396 |
}
|
| 397 |
return spec
|
| 398 |
|
|
@@ -575,13 +570,9 @@ TABLE_SPECS.update({
|
|
| 575 |
"browse_node_review_topics": _table_spec("browse_node_review_topics",
|
| 576 |
("browse_node_id", "marketplace_id", "sort_by", "sentiment", "topic"), grain="browse_node_sort_sentiment_topic"),
|
| 577 |
"sku_keywords": _table_spec("sku_keywords", ("keyword_id",),
|
| 578 |
-
(("sku", "listing_items", "sku", "many_to_one"),), grain="
|
| 579 |
"keyword_ranks": _table_spec("keyword_ranks", ("rank_id",),
|
| 580 |
(("keyword_id", "sku_keywords", "keyword_id", "many_to_one"),), grain="keyword_placement_per_sku"),
|
| 581 |
-
"competitor_snapshots": _table_spec("competitor_snapshots", ("asin", "snapshot_date"),
|
| 582 |
-
grain="asin_day"),
|
| 583 |
-
"competitor_images": _table_spec("competitor_images", ("asin", "variant", "link"),
|
| 584 |
-
grain="asin_variant_link"),
|
| 585 |
})
|
| 586 |
|
| 587 |
# A history table is one row per version, identified by its version key and first valid date.
|
|
|
|
| 95 |
"search_query_performance": "measured_at_sync",
|
| 96 |
"sku_keywords": "derived",
|
| 97 |
"keyword_ranks": "apify_dataset",
|
|
|
|
|
|
|
| 98 |
"fee_estimates_api": "measured_at_sync",
|
| 99 |
}
|
| 100 |
|
|
|
|
| 105 |
REVIEW_UNAVAILABLE_NOTE = (
|
| 106 |
"Amazon's API does not supply written reviews, average star rating or review count. "
|
| 107 |
"Read them on the product page or in Seller Central.")
|
|
|
|
| 108 |
|
| 109 |
#: Human copy for the disclosure screen, per provenance. ⛔ NO EM DASH OR EN DASH: this reaches a
|
| 110 |
#: screen and standing rule 2 governs it.
|
| 111 |
PROVENANCE_COPY = {
|
|
|
|
|
|
|
| 112 |
"apify_dataset": ("Apify Amazon search dataset joined to each SKU's latest matching keyword row on or before the capture date. "
|
| 113 |
"Sponsored rank uses the vendor rank or ordered sponsored placements in the captured dataset. "
|
| 114 |
"Global position is separate. Unmatched observations remain in the raw mirror."),
|
|
|
|
| 303 |
descriptor = descriptors[key]
|
| 304 |
label, native = descriptor.label, descriptor.native
|
| 305 |
table[key] = {"label": label, "native": native}
|
| 306 |
+
if provenance in {"derived", "apify_dataset"}:
|
| 307 |
+
table["columns"] = [dict(field) for field in AMAZON_FIELDS[key]
|
| 308 |
if field["name"] != STORE]
|
| 309 |
table["note"] = PROVENANCE_COPY[provenance]
|
| 310 |
table["local_columns"] = [dict(field) for field in AMAZON_FIELDS.get(key, ())
|
|
|
|
| 314 |
|
| 315 |
def columns_for(table):
|
| 316 |
"""The pinned column list for one table, as `[(name, type)]`. Empty when measured at sync."""
|
| 317 |
+
if PROVENANCE.get(str(table)) in {"derived", "apify_dataset"}:
|
| 318 |
return []
|
| 319 |
spec = (catalogue().get("tables") or {}).get(str(table)) or {}
|
| 320 |
return [(c["name"], c.get("type") or "VARCHAR") for c in (spec.get("columns") or [])]
|
|
|
|
| 377 |
fields = AMAZON_FIELDS.get(DATABASE_PREFIX + table, ())
|
| 378 |
spec["columns"].extend(dict(field) for field in fields if field["name"] != STORE)
|
| 379 |
spec["local_columns"] = [dict(field) for field in fields if field["name"] == STORE]
|
| 380 |
+
if PROVENANCE[table] == "derived":
|
| 381 |
+
spec["columns_source"] = {
|
| 382 |
+
"path": "platform/core/connector_spec.py", "reader": "connector_spec.AMAZON_FIELDS",
|
| 383 |
+
"provenance": "derived", "assumption": PROVENANCE_COPY["derived"],
|
| 384 |
+
"sources": ("listing_items", "search_query_performance"),
|
| 385 |
+
}
|
| 386 |
+
if PROVENANCE[table] == "apify_dataset":
|
| 387 |
spec["columns_source"] = {
|
| 388 |
"path": "platform/core/connector_spec.py", "reader": "connector_spec.AMAZON_FIELDS",
|
| 389 |
+
"provenance": "apify_dataset", "assumption": PROVENANCE_COPY["apify_dataset"],
|
| 390 |
+
"sources": ("apify:khadinakbar/amazon-search-scraper", "sku_keywords"),
|
| 391 |
}
|
| 392 |
return spec
|
| 393 |
|
|
|
|
| 570 |
"browse_node_review_topics": _table_spec("browse_node_review_topics",
|
| 571 |
("browse_node_id", "marketplace_id", "sort_by", "sentiment", "topic"), grain="browse_node_sort_sentiment_topic"),
|
| 572 |
"sku_keywords": _table_spec("sku_keywords", ("keyword_id",),
|
| 573 |
+
(("sku", "listing_items", "sku", "many_to_one"),), grain="sku_month_asin_query"),
|
| 574 |
"keyword_ranks": _table_spec("keyword_ranks", ("rank_id",),
|
| 575 |
(("keyword_id", "sku_keywords", "keyword_id", "many_to_one"),), grain="keyword_placement_per_sku"),
|
|
|
|
|
|
|
|
|
|
|
|
|
| 576 |
})
|
| 577 |
|
| 578 |
# A history table is one row per version, identified by its version key and first valid date.
|
platform/harness/amazon_store.py
CHANGED
|
The diff for this file is too large to render.
See raw diff
|
|
|
platform/harness/datastore.py
CHANGED
|
@@ -3732,7 +3732,6 @@ REPLACE_EACH_PASS = {
|
|
| 3732 |
"product_tag": {"domain": []},
|
| 3733 |
}
|
| 3734 |
REPLACE_PAGE = 5000
|
| 3735 |
-
REPLACE_READ_ATTEMPTS = 3
|
| 3736 |
|
| 3737 |
_connector_spec.register("odoo", _connector_spec.tables_for("odoo"),
|
| 3738 |
mirror=_connector_spec.ConnectorMirrorSpec(tuple(ENTITIES)))
|
|
@@ -4019,13 +4018,13 @@ def _base_domain(spec):
|
|
| 4019 |
return list(spec.get("domain") or ()) + ([("active", "in", [True, False])] if spec["archivable"] else [])
|
| 4020 |
|
| 4021 |
|
| 4022 |
-
def sync_entity(key, max_batches=200, batch=BATCH, log=print
|
| 4023 |
"""Publish a single requested entity through the same source boundary as a full pass."""
|
| 4024 |
if source_paused():
|
| 4025 |
return _sync_entity(key, max_batches=max_batches, batch=batch, log=log)
|
| 4026 |
candidate = getattr(_SYNC_TLS, "candidate", None)
|
| 4027 |
if candidate is not None:
|
| 4028 |
-
phase, pulled = _sync_one(key, max_batches=max_batches, batch=batch, log=log
|
| 4029 |
# A recurring refresh may report changed pages to a bespoke cursor, but a
|
| 4030 |
# tenant pass cannot claim completion until those private pages finish too.
|
| 4031 |
pending = _connection_pending(candidate["con"], key)
|
|
@@ -4034,7 +4033,7 @@ def sync_entity(key, max_batches=200, batch=BATCH, log=print, report=None):
|
|
| 4034 |
return phase, pulled
|
| 4035 |
rt = _resident_odoo_runtime()
|
| 4036 |
if rt is None:
|
| 4037 |
-
return _sync_one(key, max_batches=max_batches, batch=batch, log=log
|
| 4038 |
report = sync_all(entities=[key], max_batches=max_batches, log=log, batch=batch)
|
| 4039 |
return report[key]["phase"], report[key]["pulled"]
|
| 4040 |
|
|
@@ -4184,17 +4183,16 @@ def _sync_entity(key, max_batches=200, batch=BATCH, log=print):
|
|
| 4184 |
con.close()
|
| 4185 |
|
| 4186 |
|
| 4187 |
-
def _sync_one(key, max_batches=200, batch=BATCH, log=print
|
| 4188 |
"""One entity's pass: the cursor sync, the full replace, or the cursor sync then the replace."""
|
| 4189 |
_connection_model(ENTITIES[key])
|
| 4190 |
plan = REPLACE_EACH_PASS.get(key)
|
| 4191 |
if plan is None or source_paused():
|
| 4192 |
return _sync_entity(key, max_batches=max_batches, batch=batch, log=log)
|
| 4193 |
if plan.get("scope") != "internal":
|
| 4194 |
-
return _replace_entity(key, log=log
|
| 4195 |
phase, pulled = _sync_entity(key, max_batches=max_batches, batch=batch, log=log)
|
| 4196 |
-
|
| 4197 |
-
return (phase if replaced == "live" else replaced), pulled + changed
|
| 4198 |
|
| 4199 |
|
| 4200 |
def _replace_scope(con, key, plan):
|
|
@@ -4208,44 +4206,37 @@ def _replace_scope(con, key, plan):
|
|
| 4208 |
return "(" + " OR ".join(terms) + ") AND " + _connection_where()
|
| 4209 |
|
| 4210 |
|
| 4211 |
-
def
|
| 4212 |
-
"""Read
|
| 4213 |
-
fields = spec["fields"] + list(spec.get("m2m") or [])
|
| 4214 |
-
for _ in range(REPLACE_READ_ATTEMPTS):
|
| 4215 |
-
expected = int(O.get_odoo().search_count(spec["model"], domain))
|
| 4216 |
-
rows, last = [], 0
|
| 4217 |
-
while True:
|
| 4218 |
-
recs = O.search_read(spec["model"], domain + [("id", ">", last)], ["id"] + fields,
|
| 4219 |
-
limit=REPLACE_PAGE, order="id asc")
|
| 4220 |
-
if any(field not in rec for rec in recs for field in fields):
|
| 4221 |
-
raise RuntimeError(f"{key}: full read omitted a requested field. The mirror keeps its previous rows.")
|
| 4222 |
-
rows += _flatten_records(recs, spec["fields"], spec.get("m2m") or (),
|
| 4223 |
-
named_m2m=spec.get("named_m2m"), raw_m2m=spec.get("raw_m2m"), spec=spec)
|
| 4224 |
-
# Only an empty page ends a read, since Odoo may return short pages before the end.
|
| 4225 |
-
if not recs:
|
| 4226 |
-
break
|
| 4227 |
-
last = recs[-1]["id"]
|
| 4228 |
-
if len(rows) == expected:
|
| 4229 |
-
return rows
|
| 4230 |
-
after = int(O.get_odoo().search_count(spec["model"], domain))
|
| 4231 |
-
if len(rows) == after:
|
| 4232 |
-
return rows
|
| 4233 |
-
return None
|
| 4234 |
|
| 4235 |
-
|
| 4236 |
-
|
| 4237 |
-
"""Replace a complete read atomically, or hold a moving entity before changing rows or state."""
|
| 4238 |
if source_paused():
|
| 4239 |
_paused_notice(f"full read of {key}", log)
|
| 4240 |
return PAUSED_PHASE, 0
|
| 4241 |
spec, plan = ENTITIES[key], REPLACE_EACH_PASS[key]
|
| 4242 |
-
|
| 4243 |
-
|
| 4244 |
-
|
| 4245 |
-
|
| 4246 |
-
|
| 4247 |
-
|
| 4248 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4249 |
con = connect()
|
| 4250 |
try:
|
| 4251 |
_ensure_table(con, key, spec)
|
|
@@ -4337,7 +4328,6 @@ def sync_all(entities=None, max_batches=200, log=print, after_entity=None, batch
|
|
| 4337 |
with binding:
|
| 4338 |
for key in selected:
|
| 4339 |
t0 = time.time()
|
| 4340 |
-
row = out.setdefault(key, {"phase": "live", "pulled": 0, "secs": 0})
|
| 4341 |
marker = _connection_state_key(key + ".pass")
|
| 4342 |
previous = candidate["con"].execute("SELECT phase FROM _sync_state WHERE entity=?", [marker]).fetchone() if entry else None
|
| 4343 |
if entry and candidate.get("resuming") and previous == ("done",):
|
|
@@ -4346,11 +4336,11 @@ def sync_all(entities=None, max_batches=200, log=print, after_entity=None, batch
|
|
| 4346 |
if entry:
|
| 4347 |
candidate["con"].execute("DELETE FROM _sync_state WHERE entity=?", [marker])
|
| 4348 |
phase, pulled = sync_entity(key, max_batches=max_batches, log=log,
|
| 4349 |
-
**({"batch": batch} if batch != BATCH else {})
|
| 4350 |
-
**({"report": row} if candidate is not None else {}))
|
| 4351 |
if entry and phase == "live":
|
| 4352 |
candidate["con"].execute("INSERT OR REPLACE INTO _sync_state VALUES (?,?,?,?,?,?)",
|
| 4353 |
[marker, "done", 0, None, 0, time.strftime("%Y-%m-%d %H:%M:%S")])
|
|
|
|
| 4354 |
if phase != "live":
|
| 4355 |
row["phase"] = phase
|
| 4356 |
row["pulled"] += pulled
|
|
|
|
| 3732 |
"product_tag": {"domain": []},
|
| 3733 |
}
|
| 3734 |
REPLACE_PAGE = 5000
|
|
|
|
| 3735 |
|
| 3736 |
_connector_spec.register("odoo", _connector_spec.tables_for("odoo"),
|
| 3737 |
mirror=_connector_spec.ConnectorMirrorSpec(tuple(ENTITIES)))
|
|
|
|
| 4018 |
return list(spec.get("domain") or ()) + ([("active", "in", [True, False])] if spec["archivable"] else [])
|
| 4019 |
|
| 4020 |
|
| 4021 |
+
def sync_entity(key, max_batches=200, batch=BATCH, log=print):
|
| 4022 |
"""Publish a single requested entity through the same source boundary as a full pass."""
|
| 4023 |
if source_paused():
|
| 4024 |
return _sync_entity(key, max_batches=max_batches, batch=batch, log=log)
|
| 4025 |
candidate = getattr(_SYNC_TLS, "candidate", None)
|
| 4026 |
if candidate is not None:
|
| 4027 |
+
phase, pulled = _sync_one(key, max_batches=max_batches, batch=batch, log=log)
|
| 4028 |
# A recurring refresh may report changed pages to a bespoke cursor, but a
|
| 4029 |
# tenant pass cannot claim completion until those private pages finish too.
|
| 4030 |
pending = _connection_pending(candidate["con"], key)
|
|
|
|
| 4033 |
return phase, pulled
|
| 4034 |
rt = _resident_odoo_runtime()
|
| 4035 |
if rt is None:
|
| 4036 |
+
return _sync_one(key, max_batches=max_batches, batch=batch, log=log)
|
| 4037 |
report = sync_all(entities=[key], max_batches=max_batches, log=log, batch=batch)
|
| 4038 |
return report[key]["phase"], report[key]["pulled"]
|
| 4039 |
|
|
|
|
| 4183 |
con.close()
|
| 4184 |
|
| 4185 |
|
| 4186 |
+
def _sync_one(key, max_batches=200, batch=BATCH, log=print):
|
| 4187 |
"""One entity's pass: the cursor sync, the full replace, or the cursor sync then the replace."""
|
| 4188 |
_connection_model(ENTITIES[key])
|
| 4189 |
plan = REPLACE_EACH_PASS.get(key)
|
| 4190 |
if plan is None or source_paused():
|
| 4191 |
return _sync_entity(key, max_batches=max_batches, batch=batch, log=log)
|
| 4192 |
if plan.get("scope") != "internal":
|
| 4193 |
+
return _replace_entity(key, log=log)
|
| 4194 |
phase, pulled = _sync_entity(key, max_batches=max_batches, batch=batch, log=log)
|
| 4195 |
+
return phase, pulled + _replace_entity(key, log=log)[1]
|
|
|
|
| 4196 |
|
| 4197 |
|
| 4198 |
def _replace_scope(con, key, plan):
|
|
|
|
| 4206 |
return "(" + " OR ".join(terms) + ") AND " + _connection_where()
|
| 4207 |
|
| 4208 |
|
| 4209 |
+
def _replace_entity(key, log=print):
|
| 4210 |
+
"""Read every row the plan's domain matches, then swap the mirror's scope for it in one transaction.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4211 |
|
| 4212 |
+
Returns (phase, changed_rows). A failed or short read raises before any delete.
|
| 4213 |
+
"""
|
|
|
|
| 4214 |
if source_paused():
|
| 4215 |
_paused_notice(f"full read of {key}", log)
|
| 4216 |
return PAUSED_PHASE, 0
|
| 4217 |
spec, plan = ENTITIES[key], REPLACE_EACH_PASS[key]
|
| 4218 |
+
domain = list(plan["domain"])
|
| 4219 |
+
fields = spec["fields"] + list(spec.get("m2m") or [])
|
| 4220 |
+
expected = int(O.get_odoo().search_count(spec["model"], domain))
|
| 4221 |
+
rows, last = [], 0
|
| 4222 |
+
while True:
|
| 4223 |
+
recs = O.search_read(spec["model"], domain + [("id", ">", last)], ["id"] + fields,
|
| 4224 |
+
limit=REPLACE_PAGE, order="id asc")
|
| 4225 |
+
if any(field not in rec for rec in recs for field in fields):
|
| 4226 |
+
raise RuntimeError(f"{key}: full read omitted a requested field. The mirror keeps its previous rows.")
|
| 4227 |
+
rows += _flatten_records(recs, spec["fields"], spec.get("m2m") or (),
|
| 4228 |
+
named_m2m=spec.get("named_m2m"), raw_m2m=spec.get("raw_m2m"), spec=spec)
|
| 4229 |
+
# Stop on an EMPTY page: a server-side page cap returns short pages before the end.
|
| 4230 |
+
if not recs:
|
| 4231 |
+
break
|
| 4232 |
+
last = recs[-1]["id"]
|
| 4233 |
+
# Rows moved during the read when the read matches a second count. Refuse only a read
|
| 4234 |
+
# that disagrees with both counts, before any delete.
|
| 4235 |
+
if len(rows) != expected:
|
| 4236 |
+
after = int(O.get_odoo().search_count(spec["model"], domain))
|
| 4237 |
+
if len(rows) != after:
|
| 4238 |
+
raise RuntimeError(f"{key}: full read returned {len(rows)} rows, counts were "
|
| 4239 |
+
f"{expected} and {after}. The mirror keeps its previous rows.")
|
| 4240 |
con = connect()
|
| 4241 |
try:
|
| 4242 |
_ensure_table(con, key, spec)
|
|
|
|
| 4328 |
with binding:
|
| 4329 |
for key in selected:
|
| 4330 |
t0 = time.time()
|
|
|
|
| 4331 |
marker = _connection_state_key(key + ".pass")
|
| 4332 |
previous = candidate["con"].execute("SELECT phase FROM _sync_state WHERE entity=?", [marker]).fetchone() if entry else None
|
| 4333 |
if entry and candidate.get("resuming") and previous == ("done",):
|
|
|
|
| 4336 |
if entry:
|
| 4337 |
candidate["con"].execute("DELETE FROM _sync_state WHERE entity=?", [marker])
|
| 4338 |
phase, pulled = sync_entity(key, max_batches=max_batches, log=log,
|
| 4339 |
+
**({"batch": batch} if batch != BATCH else {}))
|
|
|
|
| 4340 |
if entry and phase == "live":
|
| 4341 |
candidate["con"].execute("INSERT OR REPLACE INTO _sync_state VALUES (?,?,?,?,?,?)",
|
| 4342 |
[marker, "done", 0, None, 0, time.strftime("%Y-%m-%d %H:%M:%S")])
|
| 4343 |
+
row = out.setdefault(key, {"phase": "live", "pulled": 0, "secs": 0})
|
| 4344 |
if phase != "live":
|
| 4345 |
row["phase"] = phase
|
| 4346 |
row["pulled"] += pulled
|
platform/harness/mirror_store.py
CHANGED
|
@@ -355,8 +355,7 @@ def _sync_result(connector, report):
|
|
| 355 |
completed = all(isinstance(row, dict) and row.get("phase") in {"live", "backfill"}
|
| 356 |
for row in report.values())
|
| 357 |
return completed, completed and all(row["phase"] == "live" for row in report.values())
|
| 358 |
-
if
|
| 359 |
-
and any(row.get("state") == "refused" for row in (report.get("tables") or {}).values()))):
|
| 360 |
# A problem owned by one held table publishes the rest; it never makes the pass current.
|
| 361 |
holds = for_connector(connector)._publication_holds(report, ()) if connector == "amazon" else None
|
| 362 |
if holds is None or holds["blocked"]:
|
|
@@ -381,34 +380,16 @@ def _sync_result(connector, report):
|
|
| 381 |
return completed, current
|
| 382 |
|
| 383 |
|
| 384 |
-
def odoo_moving_entities(report):
|
| 385 |
-
"""Return only typed, exhausted moving-entity holds whose cause matches their entity."""
|
| 386 |
-
if not isinstance(report, dict):
|
| 387 |
-
return {}
|
| 388 |
-
return {name: row["cause"] for name, row in (report or {}).items()
|
| 389 |
-
if re.fullmatch(r"[a-z][a-z0-9_]{0,79}", str(name)) and isinstance(row, dict)
|
| 390 |
-
and row.get("phase") == "backfill" and isinstance(row.get("cause"), dict)
|
| 391 |
-
and row["cause"].get("classifier") == "odoo_entity_moving"
|
| 392 |
-
and row["cause"].get("entity") == name
|
| 393 |
-
and type(row["cause"].get("attempts")) is int and row["cause"]["attempts"] == 3}
|
| 394 |
-
|
| 395 |
-
|
| 396 |
-
def record_sync_attempt(connector, rt):
|
| 397 |
-
"""Record writer activity without claiming a completed pass or replacing a previous refusal."""
|
| 398 |
-
with _CATCH_UP_LOCK:
|
| 399 |
-
state = _catch_up_source(rt, connector)
|
| 400 |
-
if not state["current"]:
|
| 401 |
-
state.update(pass_pending=True, attempt_at=time.time())
|
| 402 |
-
|
| 403 |
-
|
| 404 |
def tracked_sync(connector, runtime, pull, *args, _catch_up_complete=True, **kwargs):
|
| 405 |
"""Count completed passes and require a clean result before clearing startup catch-up."""
|
| 406 |
connector = connector_spec.CONNECTOR_SPEC_KEYS.get(connector, connector)
|
| 407 |
rt = runtime
|
| 408 |
if rt is None:
|
| 409 |
return pull(*args, **kwargs)
|
| 410 |
-
record_sync_attempt(connector, rt)
|
| 411 |
with _CATCH_UP_LOCK:
|
|
|
|
|
|
|
|
|
|
| 412 |
active = (connector, _tenant_key(rt.key))
|
| 413 |
_ACTIVE_PULLS[active] = _ACTIVE_PULLS.get(active, 0) + 1
|
| 414 |
try:
|
|
@@ -540,99 +521,6 @@ def _sql_literal(value):
|
|
| 540 |
return "'" + str(value).replace("'", "''") + "'"
|
| 541 |
|
| 542 |
|
| 543 |
-
def _merge_publication_entries(base, local, stored):
|
| 544 |
-
"""Merge keyed observations against their common base, naming conflicting local changes."""
|
| 545 |
-
missing = object()
|
| 546 |
-
merged, conflicts = {}, []
|
| 547 |
-
for name in sorted(set(base) | set(local) | set(stored)):
|
| 548 |
-
before, ours, theirs = (entries.get(name, missing) for entries in (base, local, stored))
|
| 549 |
-
if _same_publication_entry(ours, before):
|
| 550 |
-
value = theirs
|
| 551 |
-
elif _same_publication_entry(ours, theirs):
|
| 552 |
-
value = theirs
|
| 553 |
-
elif _same_publication_entry(theirs, before):
|
| 554 |
-
value = ours
|
| 555 |
-
else:
|
| 556 |
-
conflicts.append(name)
|
| 557 |
-
continue
|
| 558 |
-
if value is not missing:
|
| 559 |
-
merged[name] = value
|
| 560 |
-
return merged, conflicts
|
| 561 |
-
|
| 562 |
-
|
| 563 |
-
def _same_publication_entry(left, right):
|
| 564 |
-
"""A content-addressed table retains its identity when only its write timestamp differs."""
|
| 565 |
-
if isinstance(left, dict) and isinstance(right, dict):
|
| 566 |
-
return {k: v for k, v in left.items() if k != "written_at"} == {
|
| 567 |
-
k: v for k, v in right.items() if k != "written_at"}
|
| 568 |
-
return left == right
|
| 569 |
-
|
| 570 |
-
|
| 571 |
-
def _marker_freshness(row):
|
| 572 |
-
"""An invalid observation time cannot outrank a valid source-state observation."""
|
| 573 |
-
try:
|
| 574 |
-
return _utc(row[5])
|
| 575 |
-
except (ValueError, OverflowError):
|
| 576 |
-
return datetime.min.replace(tzinfo=timezone.utc)
|
| 577 |
-
|
| 578 |
-
|
| 579 |
-
def _merge_publication_markers(base, local, stored):
|
| 580 |
-
"""Merge source state independently of observation time, retaining its newest valid freshness."""
|
| 581 |
-
versions = [{row[0]: tuple(row) for row in rows} for rows in (base, local, stored)]
|
| 582 |
-
states = [{name: row[:5] for name, row in version.items()} for version in versions]
|
| 583 |
-
merged, conflicts = _merge_publication_entries(*states)
|
| 584 |
-
for name, state in merged.items():
|
| 585 |
-
matching = [version[name] for version in versions
|
| 586 |
-
if name in version and version[name][:5] == state]
|
| 587 |
-
merged[name] = max(matching, key=_marker_freshness)
|
| 588 |
-
return merged, conflicts
|
| 589 |
-
|
| 590 |
-
|
| 591 |
-
def _candidate_checkpoint(con):
|
| 592 |
-
"""Read the private candidate's common-base manifest and authoritative local markers."""
|
| 593 |
-
if not con.execute("SELECT 1 FROM information_schema.tables "
|
| 594 |
-
"WHERE table_name='_candidate_checkpoint'").fetchone():
|
| 595 |
-
return None
|
| 596 |
-
row = con.execute("SELECT document FROM _candidate_checkpoint").fetchone()
|
| 597 |
-
return json.loads(row[0]) if row else None
|
| 598 |
-
|
| 599 |
-
|
| 600 |
-
def _candidate_base_tables(checkpoint):
|
| 601 |
-
"""Keep initial local content provenance distinct from an actual saved manifest."""
|
| 602 |
-
return {**checkpoint.get("local_tables", {}), **checkpoint["base"].get("tables", {})}
|
| 603 |
-
|
| 604 |
-
|
| 605 |
-
def _save_candidate_checkpoint(con, base, markers, *, local_tables=None):
|
| 606 |
-
"""Keep per-table base provenance separate from the installed serving publication."""
|
| 607 |
-
con.execute("CREATE TABLE IF NOT EXISTS _candidate_checkpoint (document VARCHAR)")
|
| 608 |
-
con.execute("DELETE FROM _candidate_checkpoint")
|
| 609 |
-
con.execute("INSERT INTO _candidate_checkpoint VALUES (?)", [json.dumps({
|
| 610 |
-
"base": base, "markers": markers, "local_tables": local_tables or {}})])
|
| 611 |
-
|
| 612 |
-
|
| 613 |
-
def _save_initial_candidate_checkpoint(store, rt, candidate, published):
|
| 614 |
-
"""Capture pre-manifest local table content before private source work changes it."""
|
| 615 |
-
base = datastore._publication(published, store.connector)
|
| 616 |
-
local_tables = {}
|
| 617 |
-
missing = store._tables(published) - set(base.get("tables", {}))
|
| 618 |
-
if missing:
|
| 619 |
-
sealed = store._sealed_mode(rt)
|
| 620 |
-
part_limit = store.part_max_bytes - (len(be.HEADER) + 28 if sealed else 0)
|
| 621 |
-
with tempfile.TemporaryDirectory(prefix="mirror-local-base-") as tmp:
|
| 622 |
-
for name in sorted(missing):
|
| 623 |
-
entry, _fresh = store._checkpoint_table(published, name, {}, set(), tmp,
|
| 624 |
-
sealed=sealed, part_limit=part_limit)
|
| 625 |
-
local_tables[name] = entry
|
| 626 |
-
_save_candidate_checkpoint(candidate, base, store._state_rows(published), local_tables=local_tables)
|
| 627 |
-
|
| 628 |
-
|
| 629 |
-
def _table_marker_rows(store, rows, tables):
|
| 630 |
-
"""Pair held data markers and direct table status with their table's rows."""
|
| 631 |
-
names = {row[0] for row in store._held_state_rows(rows, tables)}
|
| 632 |
-
names.update(row[0] for row in rows if row[0].split("__", 1)[-1] in tables)
|
| 633 |
-
return [row for row in rows if row[0] in names]
|
| 634 |
-
|
| 635 |
-
|
| 636 |
class MirrorStore:
|
| 637 |
"""One connector policy, with state cached separately for each tenant mirror."""
|
| 638 |
|
|
@@ -1410,232 +1298,162 @@ class MirrorStore:
|
|
| 1410 |
return entry, fresh
|
| 1411 |
|
| 1412 |
|
| 1413 |
-
def _rebase_publication(self, tenant_key, rt, con, base, manifest, rows, *, allowed, report):
|
| 1414 |
-
"""Keep stored changes and their markers, holding overlapping private changes for retry."""
|
| 1415 |
-
raw = self._download(rt, _MANIFEST_KEY)
|
| 1416 |
-
if raw is None:
|
| 1417 |
-
# A local snapshot identity alone does not prove a saved manifest existed.
|
| 1418 |
-
if "tables" in base:
|
| 1419 |
-
raise PublicationHeld(f"{self.label} checkpoint held: the saved base publication is missing.")
|
| 1420 |
-
return rows, False
|
| 1421 |
-
stored = json.loads(raw)
|
| 1422 |
-
if stored.get("tenant") != _tenant_key(tenant_key):
|
| 1423 |
-
raise ValueError("Mirror manifest belongs to another tenant")
|
| 1424 |
-
has_base = con.execute("SELECT 1 FROM information_schema.tables "
|
| 1425 |
-
"WHERE table_name='_candidate_base'").fetchone()
|
| 1426 |
-
candidate_base = con.execute("SELECT identity FROM _candidate_base").fetchone() if has_base else None
|
| 1427 |
-
base_id = candidate_base[0] if candidate_base else base.get("snapshot_id", "unknown")
|
| 1428 |
-
if base_id != base.get("snapshot_id", "unknown"):
|
| 1429 |
-
raise PublicationHeld(f"{self.label} checkpoint held: private rows have a different base publication.")
|
| 1430 |
-
if (stored.get("snapshot_id", "unknown") == base_id
|
| 1431 |
-
and stored.get("tables") == base.get("tables")
|
| 1432 |
-
and stored.get("sync_state") == base.get("sync_state")):
|
| 1433 |
-
return rows, False
|
| 1434 |
-
checkpoint = _candidate_checkpoint(con)
|
| 1435 |
-
tables, conflicts = _merge_publication_entries(
|
| 1436 |
-
_candidate_base_tables(checkpoint) if checkpoint else base.get("tables", {}),
|
| 1437 |
-
manifest["tables"], stored.get("tables", {}))
|
| 1438 |
-
if not allowed:
|
| 1439 |
-
names = ", ".join(conflicts) or "the source's held tables"
|
| 1440 |
-
raise PublicationHeld(f"{self.label} checkpoint held: a newer stored publication changed {names}. "
|
| 1441 |
-
"Private rows keep their previous base for the next pass.")
|
| 1442 |
-
with tempfile.TemporaryDirectory(prefix="mirror-rebase-") as tmp:
|
| 1443 |
-
self._before_restore(tenant_key, rt)
|
| 1444 |
-
stored, stored_rows, _paths, _total, _refused = self._validated_snapshot(
|
| 1445 |
-
tenant_key, rt, con, raw, tmp)
|
| 1446 |
-
if checkpoint is None:
|
| 1447 |
-
raise PublicationHeld(f"{self.label} checkpoint held: common-base markers are unavailable.")
|
| 1448 |
-
base_rows = checkpoint["markers"]
|
| 1449 |
-
markers, marker_conflicts = _merge_publication_markers(base_rows, rows, stored_rows)
|
| 1450 |
-
versions = base_rows + rows + stored_rows
|
| 1451 |
-
conflicts = sorted(set(conflicts) | {
|
| 1452 |
-
name for name in report.get("held_tables", ())
|
| 1453 |
-
if set(marker_conflicts) & {row[0] for row in _table_marker_rows(self, versions, {name})}})
|
| 1454 |
-
held_markers = {row[0] for row in _table_marker_rows(self, versions, set(conflicts))}
|
| 1455 |
-
if set(marker_conflicts) - held_markers:
|
| 1456 |
-
raise PublicationHeld(f"{self.label} checkpoint held: a newer stored publication changed markers "
|
| 1457 |
-
+ ", ".join(marker_conflicts) + ". Private rows stay for the next pass.")
|
| 1458 |
-
if conflicts:
|
| 1459 |
-
report["conflicts"] = conflicts
|
| 1460 |
-
report["problems"].append(f"{self.label} checkpoint held tables: a newer stored publication changed "
|
| 1461 |
-
+ ", ".join(conflicts) + ". Private rows keep their previous base for the next pass.")
|
| 1462 |
-
tables.update({name: stored["tables"][name] for name in conflicts if name in stored["tables"]})
|
| 1463 |
-
markers = {name: row for name, row in markers.items() if name not in held_markers}
|
| 1464 |
-
markers.update({row[0]: tuple(row) for row in stored_rows if row[0] in held_markers})
|
| 1465 |
-
manifest["tables"] = tables
|
| 1466 |
-
return list(markers.values()), True
|
| 1467 |
-
|
| 1468 |
-
|
| 1469 |
-
def _checkpoint_table(self, con, name, prior, named, tmp, *, sealed, part_limit):
|
| 1470 |
-
"""Prepare changed immutable content, reusing every part the base already names."""
|
| 1471 |
-
path = Path(tmp) / (name + ".parquet")
|
| 1472 |
-
size = None
|
| 1473 |
-
if "parts" not in prior:
|
| 1474 |
-
con.execute(f"COPY (SELECT * FROM {_quoted_identifier(name)}) TO ? "
|
| 1475 |
-
"(FORMAT PARQUET, COMPRESSION ZSTD)", [str(path)])
|
| 1476 |
-
size = path.stat().st_size
|
| 1477 |
-
if size is None or size > part_limit:
|
| 1478 |
-
return self._chunked_entry(con, name, Path(tmp), prior, named, sealed=sealed)
|
| 1479 |
-
metadata = _blob_metadata(path.read_bytes())
|
| 1480 |
-
if (metadata["sha256"] == prior.get("sha256") and prior.get("key")
|
| 1481 |
-
and (".sealed-v1." in prior["key"]) == sealed):
|
| 1482 |
-
return None, []
|
| 1483 |
-
suffix = ".sealed-v1.parquet" if sealed else ".parquet"
|
| 1484 |
-
key = _content_key(name, suffix, metadata)
|
| 1485 |
-
metadata.update(rows=con.execute(f"SELECT count(*) FROM {_quoted_identifier(name)}").fetchone()[0],
|
| 1486 |
-
written_at=_iso(datetime.now(timezone.utc)), key=key)
|
| 1487 |
-
if key in named and named[key] != metadata["sha256"]:
|
| 1488 |
-
raise ValueError(f"{self.label} mirror content key collides with a published blob")
|
| 1489 |
-
return metadata, [] if key in named else [(key, path)]
|
| 1490 |
-
|
| 1491 |
-
|
| 1492 |
-
def _checkpoint_tables(self, rt, con, manifest, dirty, named, tmp, *, sealed, part_limit):
|
| 1493 |
-
"""Prepare local changes and migrate legacy remote entries without dropping refused tables."""
|
| 1494 |
-
tables, changed, pending = self._tables(con), [], []
|
| 1495 |
-
legacy = {name for name, meta in manifest["tables"].items()
|
| 1496 |
-
if (not meta.get("key") and "parts" not in meta)
|
| 1497 |
-
or any((".sealed-v1." in key) != sealed for key, _ in self._entry_blob_keys(name, meta))}
|
| 1498 |
-
for name in sorted(legacy - tables):
|
| 1499 |
-
prior, migrated = manifest["tables"][name], []
|
| 1500 |
-
for index, (key, metadata) in enumerate(self._entry_blob_keys(name, prior)):
|
| 1501 |
-
if (".sealed-v1." in key) == sealed:
|
| 1502 |
-
migrated.append(dict(metadata))
|
| 1503 |
-
continue
|
| 1504 |
-
path = Path(tmp) / f"{name}.migrate{index}.parquet"
|
| 1505 |
-
path.write_bytes(self._checked_mirror_blob(rt, key, metadata))
|
| 1506 |
-
new_key = key.replace(".parquet", ".sealed-v1.parquet")
|
| 1507 |
-
migrated.append({**metadata, "key": new_key})
|
| 1508 |
-
pending.append((new_key, path))
|
| 1509 |
-
manifest["tables"][name] = ({**prior, "parts": migrated} if "parts" in prior
|
| 1510 |
-
else {**prior, **migrated[0]})
|
| 1511 |
-
changed.append(name)
|
| 1512 |
-
for name in sorted((dirty | legacy | (tables - set(manifest["tables"]))) & tables):
|
| 1513 |
-
entry, fresh = self._checkpoint_table(con, name, manifest["tables"].get(name, {}), named, tmp,
|
| 1514 |
-
sealed=sealed, part_limit=part_limit)
|
| 1515 |
-
if entry is not None:
|
| 1516 |
-
manifest["tables"][name] = entry
|
| 1517 |
-
changed.append(name)
|
| 1518 |
-
pending.extend(fresh)
|
| 1519 |
-
return changed, pending
|
| 1520 |
-
|
| 1521 |
-
|
| 1522 |
-
def _upload_checkpoint_parts(self, tenant_key, rt, pending, part_limit, progress):
|
| 1523 |
-
"""Ledger immutable keys before uploading bounded parts, retaining byte progress on failure."""
|
| 1524 |
-
if any(path.stat().st_size > part_limit for _key, path in pending):
|
| 1525 |
-
raise ValueError("mirror part exceeds the byte limit")
|
| 1526 |
-
if pending:
|
| 1527 |
-
self._ledger_add(rt, tenant_key, [key for key, _path in pending])
|
| 1528 |
-
for key, path in pending:
|
| 1529 |
-
data = path.read_bytes()
|
| 1530 |
-
self._upload(rt, key, data)
|
| 1531 |
-
progress["uploaded"] += len(data)
|
| 1532 |
-
|
| 1533 |
-
|
| 1534 |
-
def _publish_checkpoint(self, tenant_key, rt, con, cache, manifest, changed, pending, tmp, *,
|
| 1535 |
-
publication, hold, rebase_allowed, report, log, progress):
|
| 1536 |
-
"""Publish one complete manifest after content upload and a fresh stored-base comparison."""
|
| 1537 |
-
checkpoint = _candidate_checkpoint(con)
|
| 1538 |
-
old = checkpoint["base"] if checkpoint else cache["manifest"]
|
| 1539 |
-
sealed = bool(manifest.get("blob_envelope_version"))
|
| 1540 |
-
part_limit = self.part_max_bytes - (len(be.HEADER) + 28 if sealed else 0)
|
| 1541 |
-
rows = self._state_rows(con)
|
| 1542 |
-
if not self._state_complete(rows):
|
| 1543 |
-
return None
|
| 1544 |
-
suffix = ".sealed-v1" if sealed else ""
|
| 1545 |
-
state_meta, _blobs = self._state_entry(rows, part_limit, suffix)
|
| 1546 |
-
if not (publication or changed or state_meta != old.get("sync_state")
|
| 1547 |
-
or manifest["tables"] != old.get("tables", {}) or old.get("version") != MANIFEST_VERSION):
|
| 1548 |
-
return None
|
| 1549 |
-
self._upload_checkpoint_parts(tenant_key, rt, pending, part_limit, progress)
|
| 1550 |
-
rows, report["rebased"] = self._rebase_publication(
|
| 1551 |
-
tenant_key, rt, con, old, manifest, rows,
|
| 1552 |
-
allowed=rebase_allowed and publication is not None and hold is None, report=report)
|
| 1553 |
-
state_meta, blobs = self._state_entry(rows, part_limit, suffix)
|
| 1554 |
-
pending, named = [], self._manifest_keys(old)
|
| 1555 |
-
for index, (key, state) in enumerate(blobs):
|
| 1556 |
-
if key not in named:
|
| 1557 |
-
path = Path(tmp) / f"sync_state.{index}.json"
|
| 1558 |
-
path.write_bytes(state)
|
| 1559 |
-
pending.append((key, path))
|
| 1560 |
-
self._upload_checkpoint_parts(tenant_key, rt, pending, part_limit, progress)
|
| 1561 |
-
stamp = _iso(datetime.now(timezone.utc))
|
| 1562 |
-
manifest.update(sync_state=state_meta, writer=_writer(), written_at=stamp, **(publication or {}))
|
| 1563 |
-
if publication is not None:
|
| 1564 |
-
manifest["completed_at"] = stamp
|
| 1565 |
-
if hold is not None:
|
| 1566 |
-
manifest["hold"] = hold
|
| 1567 |
-
self._upload(rt, _HELD_KEY, _mirror_json(manifest))
|
| 1568 |
-
return manifest
|
| 1569 |
-
superseded = _superseded_hold(self, tenant_key, rt)
|
| 1570 |
-
if superseded:
|
| 1571 |
-
manifest["superseded_hold"] = superseded
|
| 1572 |
-
published = self._upload(rt, _MANIFEST_KEY, _mirror_json(manifest))
|
| 1573 |
-
progress["uploaded"] += len(published)
|
| 1574 |
-
if superseded is not None:
|
| 1575 |
-
try:
|
| 1576 |
-
rt.delete_path(self.prefix + _HELD_KEY)
|
| 1577 |
-
except Exception as exc:
|
| 1578 |
-
log(f"{self.label} superseded hold cleanup deferred ({type(exc).__name__}); "
|
| 1579 |
-
"the next publication retries")
|
| 1580 |
-
if publication is None:
|
| 1581 |
-
datastore._write_publication(con, self.connector, manifest)
|
| 1582 |
-
cache["manifest"] = manifest
|
| 1583 |
-
cache["problems"].clear()
|
| 1584 |
-
self._retire_blobs(tenant_key, rt, published, named, log, orphans=cache.get("state") != "refused")
|
| 1585 |
-
return manifest
|
| 1586 |
-
|
| 1587 |
-
|
| 1588 |
def persist_mirror(self, tenant_key, rt, con, changed_tables, *, report, log=print,
|
| 1589 |
-
_publication=None, _hold=None
|
| 1590 |
-
"""Checkpoint
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1591 |
candidate = getattr(datastore._SYNC_TLS, "candidate", None)
|
| 1592 |
if candidate is not None and _publication is None:
|
| 1593 |
if candidate["tenant"] != tenant_key or candidate["connector"] != self.connector:
|
| 1594 |
raise ValueError("Checkpoint belongs to another source candidate")
|
| 1595 |
-
return
|
| 1596 |
-
|
| 1597 |
with self.lock:
|
| 1598 |
cache = self.restore_mirror(tenant_key, rt, log=log)
|
| 1599 |
cache["dirty"].update(changed_tables)
|
| 1600 |
try:
|
| 1601 |
if cache.get("state") == "refused" and not self._refused_may_publish(tenant_key, rt, con, cache):
|
| 1602 |
-
|
| 1603 |
-
|
| 1604 |
-
|
| 1605 |
-
|
| 1606 |
-
|
| 1607 |
-
|
|
|
|
| 1608 |
sealed = self._sealed_mode(rt)
|
| 1609 |
-
if old.get(
|
| 1610 |
-
raise tenant_keys.TenantKeyLocked(
|
|
|
|
|
|
|
| 1611 |
if old.get("snapshot_id") and _publication is None:
|
| 1612 |
raise ValueError("An identified source requires the complete sync publication boundary")
|
| 1613 |
-
|
|
|
|
|
|
|
| 1614 |
manifest = {"version": MANIFEST_VERSION, "tenant": _tenant_key(tenant_key),
|
| 1615 |
-
"tables": {name: meta for name, meta in old.get("tables", {}).items()
|
|
|
|
| 1616 |
if sealed:
|
| 1617 |
-
manifest[
|
| 1618 |
-
|
| 1619 |
with tempfile.TemporaryDirectory(prefix="mirror-persist-") as tmp:
|
| 1620 |
-
|
| 1621 |
-
|
| 1622 |
-
|
| 1623 |
-
|
| 1624 |
-
|
| 1625 |
-
|
| 1626 |
-
|
| 1627 |
-
|
| 1628 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1629 |
cache["dirty"].clear()
|
| 1630 |
-
log(f"{self.label} mirror persisted to the store: {len(changed)} changed tables, "
|
| 1631 |
-
|
| 1632 |
-
return saved
|
| 1633 |
-
except PublicationHeld as exc:
|
| 1634 |
-
report["problems"].append(str(exc))
|
| 1635 |
-
log(str(exc))
|
| 1636 |
except Exception as exc:
|
| 1637 |
-
message = (f"{self.label} mirror persistence failed ({type(exc).__name__}); "
|
| 1638 |
-
f"{progress['uploaded']:,} bytes uploaded. "
|
| 1639 |
"Local rows were kept; restore tenant-store access and resync before restarting.")
|
| 1640 |
report["problems"].append(message)
|
| 1641 |
log(message)
|
|
@@ -1820,20 +1638,8 @@ def _export_tables(store, con, directory):
|
|
| 1820 |
return paths
|
| 1821 |
|
| 1822 |
|
| 1823 |
-
def _candidate_matches(store, candidate, published, previous):
|
| 1824 |
-
"""Resume a known same-tenant common base even after another publication was installed."""
|
| 1825 |
-
if not previous:
|
| 1826 |
-
return False
|
| 1827 |
-
installed = datastore._publication(published, store.connector)
|
| 1828 |
-
checkpoint = _candidate_checkpoint(candidate)
|
| 1829 |
-
if checkpoint is None:
|
| 1830 |
-
return previous[0] == installed.get("snapshot_id", "unknown")
|
| 1831 |
-
base = checkpoint["base"]
|
| 1832 |
-
return previous[0] == base.get("snapshot_id", "unknown") and base.get("tenant") == installed.get("tenant")
|
| 1833 |
-
|
| 1834 |
-
|
| 1835 |
def _open_candidate(store, tenant_key, rt, published):
|
| 1836 |
-
"""Resume
|
| 1837 |
path = Path(rt.datastore_path).with_suffix(f".{store.connector}.candidate.duckdb")
|
| 1838 |
candidate = datastore.apply_limits(datastore.duckdb.connect(str(path)))
|
| 1839 |
datastore._ensure_sync_state(candidate)
|
|
@@ -1843,9 +1649,7 @@ def _open_candidate(store, tenant_key, rt, published):
|
|
| 1843 |
candidate.execute("CREATE TABLE IF NOT EXISTS _candidate_held (name VARCHAR PRIMARY KEY)")
|
| 1844 |
previous = candidate.execute("SELECT * FROM _candidate_base").fetchone()
|
| 1845 |
base = datastore._publication(published, store.connector).get("snapshot_id", "unknown")
|
| 1846 |
-
if
|
| 1847 |
-
if _candidate_checkpoint(candidate) is None:
|
| 1848 |
-
_save_initial_candidate_checkpoint(store, rt, candidate, published)
|
| 1849 |
return candidate, previous[1]
|
| 1850 |
started = _iso(datetime.now(timezone.utc))
|
| 1851 |
try:
|
|
@@ -1860,7 +1664,6 @@ def _open_candidate(store, tenant_key, rt, published):
|
|
| 1860 |
candidate.execute("DELETE FROM _candidate_pending")
|
| 1861 |
candidate.execute("DELETE FROM _candidate_held")
|
| 1862 |
candidate.execute("INSERT INTO _candidate_base VALUES (?,?)", [base, started])
|
| 1863 |
-
_save_initial_candidate_checkpoint(store, rt, candidate, published)
|
| 1864 |
return candidate, started
|
| 1865 |
except BaseException:
|
| 1866 |
candidate.close()
|
|
@@ -1880,7 +1683,7 @@ def pending_sync_entities(connector, tenant_key, rt):
|
|
| 1880 |
candidate = datastore.apply_limits(datastore.duckdb.connect(str(path)))
|
| 1881 |
try:
|
| 1882 |
base = candidate.execute("SELECT identity FROM _candidate_base").fetchone()
|
| 1883 |
-
if not
|
| 1884 |
return None
|
| 1885 |
pending = {row[0] for row in candidate.execute("SELECT name FROM _candidate_pending").fetchall()}
|
| 1886 |
if store.connector == "odoo":
|
|
@@ -1898,13 +1701,6 @@ def _holds_rows(holds):
|
|
| 1898 |
return holds is not None and bool(holds["tables"] or holds.get("accounts"))
|
| 1899 |
|
| 1900 |
|
| 1901 |
-
def _held_tables(holds):
|
| 1902 |
-
"""Retain whole private tables and their original bases even when only an account is held."""
|
| 1903 |
-
if holds is None:
|
| 1904 |
-
return set()
|
| 1905 |
-
return set(holds["tables"]).union(*(hold["tables"] for hold in (holds.get("accounts") or {}).values()))
|
| 1906 |
-
|
| 1907 |
-
|
| 1908 |
def _hold_previous(store, candidate, published, tables):
|
| 1909 |
"""Give each held table its published rows and data markers, inside the candidate's open transaction."""
|
| 1910 |
prior = store._tables(published)
|
|
@@ -1925,36 +1721,6 @@ def _hold_previous(store, candidate, published, tables):
|
|
| 1925 |
candidate.executemany("INSERT INTO _sync_state VALUES (?,?,?,?,?,?)", previous)
|
| 1926 |
|
| 1927 |
|
| 1928 |
-
def _retain_conflicting_candidate(store, candidate, published, manifest, conflicts):
|
| 1929 |
-
"""Refresh independent tables while retaining held private rows, markers and their original common bases."""
|
| 1930 |
-
checkpoint = _candidate_checkpoint(candidate)
|
| 1931 |
-
tables = set(conflicts)
|
| 1932 |
-
private = _table_marker_rows(store, store._state_rows(candidate), tables)
|
| 1933 |
-
saved_rows = store._state_rows(published)
|
| 1934 |
-
held_names = {row[0] for row in _table_marker_rows(store, checkpoint["markers"] + saved_rows, tables)}
|
| 1935 |
-
base_rows = [row for row in saved_rows if row[0] not in held_names]
|
| 1936 |
-
base_rows += [row for row in checkpoint["markers"] if row[0] in held_names]
|
| 1937 |
-
base = {**manifest, "snapshot_id": checkpoint["base"].get("snapshot_id", "unknown"),
|
| 1938 |
-
"tables": {name: entry for name, entry in manifest["tables"].items() if name not in tables}}
|
| 1939 |
-
base["tables"].update({name: entry for name, entry in _candidate_base_tables(checkpoint).items()
|
| 1940 |
-
if name in tables})
|
| 1941 |
-
candidate.execute("BEGIN TRANSACTION")
|
| 1942 |
-
try:
|
| 1943 |
-
_hold_previous(store, candidate, published, (store._tables(candidate) | store._tables(published)) - tables)
|
| 1944 |
-
owned = store._state_rows(candidate)
|
| 1945 |
-
if owned:
|
| 1946 |
-
candidate.executemany("DELETE FROM _sync_state WHERE entity=?", [[row[0]] for row in owned])
|
| 1947 |
-
rows = [row for row in saved_rows if row[0] not in held_names] + private
|
| 1948 |
-
if rows:
|
| 1949 |
-
candidate.executemany("INSERT INTO _sync_state VALUES (?,?,?,?,?,?)", rows)
|
| 1950 |
-
_save_candidate_checkpoint(candidate, base, base_rows)
|
| 1951 |
-
candidate.executemany("INSERT OR IGNORE INTO _candidate_held VALUES (?)", [[name] for name in conflicts])
|
| 1952 |
-
candidate.execute("COMMIT")
|
| 1953 |
-
except BaseException:
|
| 1954 |
-
candidate.execute("ROLLBACK")
|
| 1955 |
-
raise
|
| 1956 |
-
|
| 1957 |
-
|
| 1958 |
def _publication_line(store, report, done, holds, *, snapshot_id=None, why=None):
|
| 1959 |
"""One log line: what a pass published and what it held, each with its reason."""
|
| 1960 |
rows = report.get("tables") or {}
|
|
@@ -2124,7 +1890,7 @@ def sync_mirror(connector, tenant_key, rt, *, report=None, log=print, before_pub
|
|
| 2124 |
prior_snapshot_id = datastore._publication(published, store.connector).get("snapshot_id", "unknown")
|
| 2125 |
metadata.pop("rows")
|
| 2126 |
metadata["completed_at"] = _iso(datetime.now(timezone.utc))
|
| 2127 |
-
result = {"problems": []
|
| 2128 |
save_started = time.monotonic()
|
| 2129 |
with tempfile.TemporaryDirectory(prefix="mirror-publish-") as tmp:
|
| 2130 |
exporting = time.monotonic()
|
|
@@ -2142,18 +1908,9 @@ def sync_mirror(connector, tenant_key, rt, *, report=None, log=print, before_pub
|
|
| 2142 |
report["problems"].extend(problems)
|
| 2143 |
return
|
| 2144 |
raise RuntimeError("; ".join(problems))
|
| 2145 |
-
if report is not None and "problems" in report:
|
| 2146 |
-
report["problems"].extend(result["problems"])
|
| 2147 |
-
retained = set(result.get("conflicts", ())) | result["held_tables"]
|
| 2148 |
-
kept = bool(retained)
|
| 2149 |
try:
|
| 2150 |
installing = time.monotonic()
|
| 2151 |
-
|
| 2152 |
-
if result.get("rebased"):
|
| 2153 |
-
store._before_restore(tenant_key, rt)
|
| 2154 |
-
_saved, rows, paths, _total, _refused = store._validated_snapshot(
|
| 2155 |
-
tenant_key, rt, published, _mirror_json(manifest), tmp)
|
| 2156 |
-
store._install_snapshot(published, manifest, rows, paths)
|
| 2157 |
installed = True
|
| 2158 |
store._after_publish(tenant_key, manifest)
|
| 2159 |
except BaseException:
|
|
@@ -2163,19 +1920,20 @@ def sync_mirror(connector, tenant_key, rt, *, report=None, log=print, before_pub
|
|
| 2163 |
note("install", installing)
|
| 2164 |
for listener in list(_PUBLISH_LISTENERS):
|
| 2165 |
try:
|
| 2166 |
-
listener(store.connector, tenant_key, manifest,
|
| 2167 |
except Exception as exc:
|
| 2168 |
# The snapshot is saved and installed here. A failed hand-off only delays readers.
|
| 2169 |
log(f"{store.label} mirror hand-off failed ({type(exc).__name__}: {exc})")
|
| 2170 |
if _holds_rows(holds):
|
| 2171 |
candidate.execute("ROLLBACK")
|
| 2172 |
-
|
| 2173 |
-
|
| 2174 |
-
if odoo
|
| 2175 |
# D-11329: a held entity keeps its private progress but never narrows the next pass.
|
| 2176 |
names = [[name] for name in sorted(holds["held"])]
|
| 2177 |
candidate.executemany("DELETE FROM _candidate_pending WHERE name=?", names)
|
| 2178 |
candidate.executemany("INSERT OR IGNORE INTO _candidate_held VALUES (?)", names)
|
|
|
|
| 2179 |
if holds is not None:
|
| 2180 |
log(_publication_line(store, report, done, holds, snapshot_id=manifest["snapshot_id"]))
|
| 2181 |
if odoo:
|
|
|
|
| 355 |
completed = all(isinstance(row, dict) and row.get("phase") in {"live", "backfill"}
|
| 356 |
for row in report.values())
|
| 357 |
return completed, completed and all(row["phase"] == "live" for row in report.values())
|
| 358 |
+
if report.get("problems") or report.get("audit_problems"):
|
|
|
|
| 359 |
# A problem owned by one held table publishes the rest; it never makes the pass current.
|
| 360 |
holds = for_connector(connector)._publication_holds(report, ()) if connector == "amazon" else None
|
| 361 |
if holds is None or holds["blocked"]:
|
|
|
|
| 380 |
return completed, current
|
| 381 |
|
| 382 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 383 |
def tracked_sync(connector, runtime, pull, *args, _catch_up_complete=True, **kwargs):
|
| 384 |
"""Count completed passes and require a clean result before clearing startup catch-up."""
|
| 385 |
connector = connector_spec.CONNECTOR_SPEC_KEYS.get(connector, connector)
|
| 386 |
rt = runtime
|
| 387 |
if rt is None:
|
| 388 |
return pull(*args, **kwargs)
|
|
|
|
| 389 |
with _CATCH_UP_LOCK:
|
| 390 |
+
state = _catch_up_source(rt, connector)
|
| 391 |
+
if not state["current"]:
|
| 392 |
+
state.update(pass_pending=True, attempt_at=time.time())
|
| 393 |
active = (connector, _tenant_key(rt.key))
|
| 394 |
_ACTIVE_PULLS[active] = _ACTIVE_PULLS.get(active, 0) + 1
|
| 395 |
try:
|
|
|
|
| 521 |
return "'" + str(value).replace("'", "''") + "'"
|
| 522 |
|
| 523 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 524 |
class MirrorStore:
|
| 525 |
"""One connector policy, with state cached separately for each tenant mirror."""
|
| 526 |
|
|
|
|
| 1298 |
return entry, fresh
|
| 1299 |
|
| 1300 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1301 |
def persist_mirror(self, tenant_key, rt, con, changed_tables, *, report, log=print,
|
| 1302 |
+
_publication=None, _hold=None):
|
| 1303 |
+
"""Checkpoint changed tables and the connector's markers, publishing the manifest last.
|
| 1304 |
+
|
| 1305 |
+
Candidates include partially written tables. Hashing their parquet locally avoids reuploading
|
| 1306 |
+
identical content; no previous blobs are downloaded on a sync pass. Failed candidates remain
|
| 1307 |
+
dirty for retry on the next pass, while landed local rows and cursors stay usable.
|
| 1308 |
+
|
| 1309 |
+
S67-T32 (D-3481), the order that makes a killed checkpoint harmless: (1) every new blob gets a
|
| 1310 |
+
CONTENT key, and a key the current manifest names is never written; (2) the key ledger lists
|
| 1311 |
+
every new key before its upload; (3) the blobs upload; (4) ONE manifest upload, a single-row
|
| 1312 |
+
upsert, publishes them; (5) only then are superseded blobs deleted. Dying anywhere before (4)
|
| 1313 |
+
leaves the previous manifest naming only complete, untouched blobs.
|
| 1314 |
+
"""
|
| 1315 |
candidate = getattr(datastore._SYNC_TLS, "candidate", None)
|
| 1316 |
if candidate is not None and _publication is None:
|
| 1317 |
if candidate["tenant"] != tenant_key or candidate["connector"] != self.connector:
|
| 1318 |
raise ValueError("Checkpoint belongs to another source candidate")
|
| 1319 |
+
return # The enclosing source pass owns publication after every table completes.
|
| 1320 |
+
uploaded = 0
|
| 1321 |
with self.lock:
|
| 1322 |
cache = self.restore_mirror(tenant_key, rt, log=log)
|
| 1323 |
cache["dirty"].update(changed_tables)
|
| 1324 |
try:
|
| 1325 |
if cache.get("state") == "refused" and not self._refused_may_publish(tenant_key, rt, con, cache):
|
| 1326 |
+
message = (f"{self.label} mirror checkpoint held: this process could not restore the saved snapshot "
|
| 1327 |
+
"at start. The saved copy is kept; restore tenant-store access and restart to "
|
| 1328 |
+
"restore it before checkpointing.")
|
| 1329 |
+
report["problems"].append(message)
|
| 1330 |
+
log(message)
|
| 1331 |
+
return
|
| 1332 |
+
old = cache["manifest"]
|
| 1333 |
sealed = self._sealed_mode(rt)
|
| 1334 |
+
if old.get('blob_envelope_version') and not sealed:
|
| 1335 |
+
raise tenant_keys.TenantKeyLocked('A sealed mirror cannot publish without its tenant key.')
|
| 1336 |
+
suffix = '.sealed-v1' if sealed else ''
|
| 1337 |
+
part_limit = self.part_max_bytes - (len(be.HEADER) + 28 if sealed else 0)
|
| 1338 |
if old.get("snapshot_id") and _publication is None:
|
| 1339 |
raise ValueError("An identified source requires the complete sync publication boundary")
|
| 1340 |
+
named = self._manifest_keys(old)
|
| 1341 |
+
tables = self._tables(con)
|
| 1342 |
+
refused_tables = set(cache.get("refused_tables", ()))
|
| 1343 |
manifest = {"version": MANIFEST_VERSION, "tenant": _tenant_key(tenant_key),
|
| 1344 |
+
"tables": {name: meta for name, meta in old.get("tables", {}).items()
|
| 1345 |
+
if name in tables or name in refused_tables}}
|
| 1346 |
if sealed:
|
| 1347 |
+
manifest['blob_envelope_version'] = be.VERSION
|
| 1348 |
+
changed, pending = [], []
|
| 1349 |
with tempfile.TemporaryDirectory(prefix="mirror-persist-") as tmp:
|
| 1350 |
+
# A legacy entry (no content key) is a candidate too: the first checkpoint migrates.
|
| 1351 |
+
legacy = {name for name, meta in manifest["tables"].items()
|
| 1352 |
+
if (not meta.get("key") and "parts" not in meta)
|
| 1353 |
+
or any((".sealed-v1." in key) != sealed
|
| 1354 |
+
for key, _ in self._entry_blob_keys(name, meta))}
|
| 1355 |
+
# Refused local tables still name good remote parts: migrate those bytes too.
|
| 1356 |
+
for name in sorted(legacy - tables):
|
| 1357 |
+
prior = manifest['tables'][name]
|
| 1358 |
+
migrated = []
|
| 1359 |
+
for index, (key, metadata) in enumerate(self._entry_blob_keys(name, prior)):
|
| 1360 |
+
if ('.sealed-v1.' in key) == sealed:
|
| 1361 |
+
migrated.append(dict(metadata))
|
| 1362 |
+
continue
|
| 1363 |
+
data = self._checked_mirror_blob(rt, key, metadata)
|
| 1364 |
+
path = Path(tmp) / f'{name}.migrate{index}.parquet'
|
| 1365 |
+
path.write_bytes(data)
|
| 1366 |
+
new_key = key.replace('.parquet', '.sealed-v1.parquet')
|
| 1367 |
+
migrated.append({**metadata, 'key': new_key})
|
| 1368 |
+
pending.append((new_key, path))
|
| 1369 |
+
manifest['tables'][name] = ({**prior, 'parts': migrated} if 'parts' in prior
|
| 1370 |
+
else {**prior, **migrated[0]})
|
| 1371 |
+
changed.append(name)
|
| 1372 |
+
for name in sorted((cache["dirty"] | legacy | (tables - set(manifest["tables"]))) & tables):
|
| 1373 |
+
if not self._valid_table(name):
|
| 1374 |
+
raise ValueError(f"{self.label} mirror table name is invalid")
|
| 1375 |
+
prior = manifest["tables"].get(name, {})
|
| 1376 |
+
path = Path(tmp) / (name + ".parquet")
|
| 1377 |
+
size = None
|
| 1378 |
+
if "parts" not in prior:
|
| 1379 |
+
con.execute(f"COPY (SELECT * FROM {_quoted_identifier(name)}) TO ? "
|
| 1380 |
+
"(FORMAT PARQUET, COMPRESSION ZSTD)", [str(path)])
|
| 1381 |
+
size = path.stat().st_size
|
| 1382 |
+
if size is None or size > part_limit:
|
| 1383 |
+
entry, fresh = self._chunked_entry(con, name, Path(tmp), prior, named, sealed=sealed)
|
| 1384 |
+
if entry is None:
|
| 1385 |
+
continue
|
| 1386 |
+
manifest["tables"][name] = entry
|
| 1387 |
+
changed.append(name)
|
| 1388 |
+
pending.extend(fresh)
|
| 1389 |
+
continue
|
| 1390 |
+
data = path.read_bytes()
|
| 1391 |
+
metadata = _blob_metadata(data)
|
| 1392 |
+
if (metadata["sha256"] == prior.get("sha256") and prior.get("key")
|
| 1393 |
+
and (".sealed-v1." in prior["key"]) == sealed):
|
| 1394 |
+
continue
|
| 1395 |
+
key = _content_key(name, suffix + ".parquet", metadata)
|
| 1396 |
+
metadata.update(rows=con.execute(f"SELECT count(*) FROM {_quoted_identifier(name)}").fetchone()[0],
|
| 1397 |
+
written_at=_iso(datetime.now(timezone.utc)), key=key)
|
| 1398 |
+
manifest["tables"][name] = metadata
|
| 1399 |
+
changed.append(name)
|
| 1400 |
+
if key in named:
|
| 1401 |
+
# The same content under its own name is already stored and published.
|
| 1402 |
+
if named[key] != metadata["sha256"]:
|
| 1403 |
+
raise ValueError(f"{self.label} mirror content key collides with a published blob")
|
| 1404 |
+
continue
|
| 1405 |
+
pending.append((key, path))
|
| 1406 |
+
rows = self._state_rows(con)
|
| 1407 |
+
state_meta, state_blobs = self._state_entry(rows, part_limit, suffix)
|
| 1408 |
+
published = None
|
| 1409 |
+
if self._state_complete(rows) and (_publication or changed or state_meta != old.get("sync_state")
|
| 1410 |
+
or manifest["tables"] != old.get("tables", {})
|
| 1411 |
+
or old.get("version") != MANIFEST_VERSION):
|
| 1412 |
+
for index, (state_key, state) in enumerate(state_blobs):
|
| 1413 |
+
if state_key not in named:
|
| 1414 |
+
state_path = Path(tmp) / f"sync_state.{index}.json"
|
| 1415 |
+
state_path.write_bytes(state)
|
| 1416 |
+
pending.append((state_key, state_path))
|
| 1417 |
+
if any(path.stat().st_size > part_limit for _key, path in pending):
|
| 1418 |
+
raise ValueError("mirror part exceeds the byte limit")
|
| 1419 |
+
if pending:
|
| 1420 |
+
self._ledger_add(rt, tenant_key, [key for key, _path in pending])
|
| 1421 |
+
for key, path in pending:
|
| 1422 |
+
data = path.read_bytes()
|
| 1423 |
+
self._upload(rt, key, data)
|
| 1424 |
+
uploaded += len(data)
|
| 1425 |
+
stamp = _iso(datetime.now(timezone.utc))
|
| 1426 |
+
manifest.update(sync_state=state_meta, writer=_writer(), written_at=stamp,
|
| 1427 |
+
**(_publication or {}))
|
| 1428 |
+
if _publication is not None:
|
| 1429 |
+
manifest["completed_at"] = stamp
|
| 1430 |
+
if _hold is not None:
|
| 1431 |
+
manifest["hold"] = _hold
|
| 1432 |
+
self._upload(rt, _HELD_KEY, _mirror_json(manifest))
|
| 1433 |
+
return manifest
|
| 1434 |
+
superseded = _superseded_hold(self, tenant_key, rt)
|
| 1435 |
+
if superseded:
|
| 1436 |
+
manifest["superseded_hold"] = superseded
|
| 1437 |
+
published = self._upload(rt, _MANIFEST_KEY, _mirror_json(manifest))
|
| 1438 |
+
uploaded += len(published)
|
| 1439 |
+
if superseded is not None:
|
| 1440 |
+
# A newer publication replaces a stored hold. Nobody has to release it.
|
| 1441 |
+
try:
|
| 1442 |
+
rt.delete_path(self.prefix + _HELD_KEY)
|
| 1443 |
+
except Exception: # noqa: BLE001
|
| 1444 |
+
log(f"{self.label} superseded hold pointer stays until the next publication")
|
| 1445 |
+
if _publication is None:
|
| 1446 |
+
datastore._write_publication(con, self.connector, manifest)
|
| 1447 |
+
cache["manifest"] = manifest
|
| 1448 |
+
cache["problems"].clear()
|
| 1449 |
+
if published:
|
| 1450 |
+
self._retire_blobs(tenant_key, rt, published, named, log, orphans=cache.get("state") != "refused")
|
| 1451 |
+
if published or not changed:
|
| 1452 |
cache["dirty"].clear()
|
| 1453 |
+
log(f"{self.label} mirror persisted to the store: {len(changed)} changed tables, {uploaded:,} bytes uploaded")
|
| 1454 |
+
return manifest if published else None
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1455 |
except Exception as exc:
|
| 1456 |
+
message = (f"{self.label} mirror persistence failed ({type(exc).__name__}); {uploaded:,} bytes uploaded. "
|
|
|
|
| 1457 |
"Local rows were kept; restore tenant-store access and resync before restarting.")
|
| 1458 |
report["problems"].append(message)
|
| 1459 |
log(message)
|
|
|
|
| 1638 |
return paths
|
| 1639 |
|
| 1640 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1641 |
def _open_candidate(store, tenant_key, rt, published):
|
| 1642 |
+
"""Resume private rows only while their base publication still matches the serving copy."""
|
| 1643 |
path = Path(rt.datastore_path).with_suffix(f".{store.connector}.candidate.duckdb")
|
| 1644 |
candidate = datastore.apply_limits(datastore.duckdb.connect(str(path)))
|
| 1645 |
datastore._ensure_sync_state(candidate)
|
|
|
|
| 1649 |
candidate.execute("CREATE TABLE IF NOT EXISTS _candidate_held (name VARCHAR PRIMARY KEY)")
|
| 1650 |
previous = candidate.execute("SELECT * FROM _candidate_base").fetchone()
|
| 1651 |
base = datastore._publication(published, store.connector).get("snapshot_id", "unknown")
|
| 1652 |
+
if previous and previous[0] == base:
|
|
|
|
|
|
|
| 1653 |
return candidate, previous[1]
|
| 1654 |
started = _iso(datetime.now(timezone.utc))
|
| 1655 |
try:
|
|
|
|
| 1664 |
candidate.execute("DELETE FROM _candidate_pending")
|
| 1665 |
candidate.execute("DELETE FROM _candidate_held")
|
| 1666 |
candidate.execute("INSERT INTO _candidate_base VALUES (?,?)", [base, started])
|
|
|
|
| 1667 |
return candidate, started
|
| 1668 |
except BaseException:
|
| 1669 |
candidate.close()
|
|
|
|
| 1683 |
candidate = datastore.apply_limits(datastore.duckdb.connect(str(path)))
|
| 1684 |
try:
|
| 1685 |
base = candidate.execute("SELECT identity FROM _candidate_base").fetchone()
|
| 1686 |
+
if not base or base[0] != datastore._publication(published, store.connector).get("snapshot_id", "unknown"):
|
| 1687 |
return None
|
| 1688 |
pending = {row[0] for row in candidate.execute("SELECT name FROM _candidate_pending").fetchall()}
|
| 1689 |
if store.connector == "odoo":
|
|
|
|
| 1701 |
return holds is not None and bool(holds["tables"] or holds.get("accounts"))
|
| 1702 |
|
| 1703 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1704 |
def _hold_previous(store, candidate, published, tables):
|
| 1705 |
"""Give each held table its published rows and data markers, inside the candidate's open transaction."""
|
| 1706 |
prior = store._tables(published)
|
|
|
|
| 1721 |
candidate.executemany("INSERT INTO _sync_state VALUES (?,?,?,?,?,?)", previous)
|
| 1722 |
|
| 1723 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1724 |
def _publication_line(store, report, done, holds, *, snapshot_id=None, why=None):
|
| 1725 |
"""One log line: what a pass published and what it held, each with its reason."""
|
| 1726 |
rows = report.get("tables") or {}
|
|
|
|
| 1890 |
prior_snapshot_id = datastore._publication(published, store.connector).get("snapshot_id", "unknown")
|
| 1891 |
metadata.pop("rows")
|
| 1892 |
metadata["completed_at"] = _iso(datetime.now(timezone.utc))
|
| 1893 |
+
result = {"problems": []}
|
| 1894 |
save_started = time.monotonic()
|
| 1895 |
with tempfile.TemporaryDirectory(prefix="mirror-publish-") as tmp:
|
| 1896 |
exporting = time.monotonic()
|
|
|
|
| 1908 |
report["problems"].extend(problems)
|
| 1909 |
return
|
| 1910 |
raise RuntimeError("; ".join(problems))
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1911 |
try:
|
| 1912 |
installing = time.monotonic()
|
| 1913 |
+
store._install_snapshot(published, manifest, store._state_rows(candidate), paths)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1914 |
installed = True
|
| 1915 |
store._after_publish(tenant_key, manifest)
|
| 1916 |
except BaseException:
|
|
|
|
| 1920 |
note("install", installing)
|
| 1921 |
for listener in list(_PUBLISH_LISTENERS):
|
| 1922 |
try:
|
| 1923 |
+
listener(store.connector, tenant_key, manifest, store._state_rows(candidate), paths)
|
| 1924 |
except Exception as exc:
|
| 1925 |
# The snapshot is saved and installed here. A failed hand-off only delays readers.
|
| 1926 |
log(f"{store.label} mirror hand-off failed ({type(exc).__name__}: {exc})")
|
| 1927 |
if _holds_rows(holds):
|
| 1928 |
candidate.execute("ROLLBACK")
|
| 1929 |
+
candidate.execute("DELETE FROM _candidate_base")
|
| 1930 |
+
candidate.execute("INSERT INTO _candidate_base VALUES (?,?)", [manifest["snapshot_id"], started])
|
| 1931 |
+
if odoo:
|
| 1932 |
# D-11329: a held entity keeps its private progress but never narrows the next pass.
|
| 1933 |
names = [[name] for name in sorted(holds["held"])]
|
| 1934 |
candidate.executemany("DELETE FROM _candidate_pending WHERE name=?", names)
|
| 1935 |
candidate.executemany("INSERT OR IGNORE INTO _candidate_held VALUES (?)", names)
|
| 1936 |
+
kept = True
|
| 1937 |
if holds is not None:
|
| 1938 |
log(_publication_line(store, report, done, holds, snapshot_id=manifest["snapshot_id"]))
|
| 1939 |
if odoo:
|
platform/modules/amazon_profit.py
CHANGED
|
@@ -31,11 +31,9 @@ import re
|
|
| 31 |
import shutil
|
| 32 |
import tempfile
|
| 33 |
import threading
|
| 34 |
-
from decimal import Decimal, ROUND_HALF_UP
|
| 35 |
from pathlib import Path
|
| 36 |
|
| 37 |
from modules import amazon_sku_metrics as M
|
| 38 |
-
from modules import amazon_part_dates as _part_dates
|
| 39 |
import core.user_tables as _registry
|
| 40 |
from core import module_tables
|
| 41 |
from core import amazon_rivals as _rivals
|
|
@@ -121,58 +119,14 @@ def _field(key, label, ftype, description, *, computed, **extra):
|
|
| 121 |
return out
|
| 122 |
|
| 123 |
|
| 124 |
-
def _progress_fields(description
|
| 125 |
"""Project progress and its completed-count label, written by the Projects doors."""
|
| 126 |
-
return (_field(
|
| 127 |
-
format={"display": "progress", "labelKey":
|
| 128 |
-
_field(
|
| 129 |
-
"Completed projects of the counted ones." if key == "projects" else "Open PDP workflow stage and progress.",
|
| 130 |
computed=True))
|
| 131 |
|
| 132 |
|
| 133 |
-
HISTORY_FIELD = _field("history", "History", "text", "Open the row's recorded changes.", computed=True)
|
| 134 |
-
ASIN_APP_KEYS = ("asin", "projects", "pdp_update", "trend_score", "search_trend", "market_share_trend",
|
| 135 |
-
"click_share_trend", "exposure", "funnel_score", "pdp_last_updated", "title_last_updated",
|
| 136 |
-
"bullets_last_updated", "images_last_updated", "aplus_last_updated", "history")
|
| 137 |
-
SEARCH_TRENDS = (("search", "search_trend", "Search trend", "int"),
|
| 138 |
-
("market_share", "market_share_trend", "Market share trend", "pct"),
|
| 139 |
-
("click_share", "click_share_trend", "Click share trend", "pct"))
|
| 140 |
-
EXPOSURES = {"under": "Under-exposed", "match": "Matched", "over": "Over-exposed",
|
| 141 |
-
"few": "Few", "none": "Not seen"}
|
| 142 |
-
PDP_PART_DATE_KEYS = tuple(part + "_last_updated" for part in ("title", "bullets", "images", "aplus"))
|
| 143 |
-
PDP_CELL_KEYS = ("pdp_update", "pdp_update_label", "pdp_update_more") + PDP_PART_DATE_KEYS
|
| 144 |
-
_PDP_PROGRESS_FIELDS = _progress_fields("Progress of the earliest open PDP workflow.", key="pdp_update",
|
| 145 |
-
label="PDP update", empty_label="No open workflow")
|
| 146 |
-
_SIGNAL_FIELDS = (
|
| 147 |
-
_field("trend_score", "Trend score", "int", "Search trend score from 0 to 100.", computed=True),
|
| 148 |
-
*(_field(key, label, "pct", "Change against the same reported months last year.", computed=True)
|
| 149 |
-
for _, key, label, _ in SEARCH_TRENDS),
|
| 150 |
-
_field("exposure", "Exposure", "select", "Sales or click share against impression share.",
|
| 151 |
-
computed=True, options=list(EXPOSURES.values())),
|
| 152 |
-
_field("funnel_score", "Funnel score", "int", "Search-weighted funnel score, with 5 at the market.",
|
| 153 |
-
computed=True, format={"decimals": 1}),
|
| 154 |
-
)
|
| 155 |
-
_ASIN_DATE_FIELDS = tuple(_field(key, label, "date", "Latest recorded listing or part change.", computed=True)
|
| 156 |
-
for key, label in zip(("pdp_last_updated",) + PDP_PART_DATE_KEYS,
|
| 157 |
-
("PDP last updated", "Title last updated", "Bullets last updated",
|
| 158 |
-
"Images last updated", "A+ last updated")))
|
| 159 |
-
_SIGNAL_COMPANIONS = tuple(field for kind, key, label, now_type in SEARCH_TRENDS for field in (
|
| 160 |
-
_field(key + "_series", label + " series", "text", "Every reported historical month, with raw values, ASIN gaps and reasons.", computed=True),
|
| 161 |
-
_field(key + "_now", label + " now", now_type, "Searches in the newest complete store month." if kind == "search"
|
| 162 |
-
else "Share aggregated over the twelve-month calculation window.", computed=True),
|
| 163 |
-
_field(key + "_reason", label + " reason", "text", "Why the year comparison is unavailable.", computed=True),
|
| 164 |
-
_field(key + "_now_reason", label + " now reason", "text", "Why the current value is unavailable.", computed=True),
|
| 165 |
-
)) + tuple(_field(key + "_reason", label + " reason", "text", "Why this signal is unavailable.", computed=True)
|
| 166 |
-
for key, label in (("trend_score", "Trend score"), ("funnel_score", "Funnel score"), ("exposure", "Exposure")))
|
| 167 |
-
_ASIN_COMPANIONS = (_PDP_PROGRESS_FIELDS[1:]
|
| 168 |
-
+ (_field("pdp_update_more", "More PDP workflows", "int", "Other open PDP workflows.", computed=True),)
|
| 169 |
-
+ _SIGNAL_COMPANIONS
|
| 170 |
-
+ (_field("exposure_label", "Exposure detail", "text", "Exposure multiple and comparison base.", computed=True),
|
| 171 |
-
_field("pdp_last_updated_reason", "PDP date reason", "text", "Why Amazon's listing date is unavailable.", computed=True)))
|
| 172 |
-
_ASIN_SOURCE_KEYS = tuple(f["key"] for f in _SIGNAL_FIELDS + _SIGNAL_COMPANIONS) + (
|
| 173 |
-
"exposure_label", "pdp_last_updated", "pdp_last_updated_reason")
|
| 174 |
-
|
| 175 |
-
|
| 176 |
def _metric_fields(skip=()):
|
| 177 |
"""The sibling metric module's columns, as computed fields of this module."""
|
| 178 |
return tuple(_field(key, label, ftype, description, computed=True, **dict(extra or {}))
|
|
@@ -226,7 +180,7 @@ PARENT_COMPETITOR_FIELDS = tuple(_field(k, label, t, d, computed=True, **dict(x)
|
|
| 226 |
for k, label, t, d, x in _rivals.PARENT_COLUMNS)
|
| 227 |
COMPETITOR_KEYS = _rivals.ROW_KEYS + _rivals.PARENT_KEYS
|
| 228 |
_SKU_PROJECT_FIELDS = _progress_fields("Average progress of the projects on this SKU, its ASIN and its parent ASIN.")
|
| 229 |
-
COMPUTED_FIELDS = (_ECON_FIELDS[:1] + _SKU_PROJECT_FIELDS[:1] + (
|
| 230 |
+ (HEALTH_NOTE_FIELD,) + _SKU_PROJECT_FIELDS[1:]
|
| 231 |
+ COMPETITOR_FIELDS)
|
| 232 |
#: Owned columns: people edit them and a refresh never writes them.
|
|
@@ -378,7 +332,7 @@ def cost_note(row, *, code=None, found=True):
|
|
| 378 |
if source and row.get("cogs_per_unit") is None:
|
| 379 |
return source if source.endswith(".") else source + "."
|
| 380 |
return NO_COST_NOTE
|
| 381 |
-
_IDENTITY_KEYS = ("sku", "picture", "asin", "parent_asin", "title", "projects", "projects_label"
|
| 382 |
_PERIOD_KEYS = tuple(k for k in COMPUTED_KEYS if k not in _IDENTITY_KEYS + COMPETITOR_KEYS)
|
| 383 |
|
| 384 |
#: The profit block a rollup table carries; `_ECON_ADDITIVE` sums, margin and band recompute.
|
|
@@ -397,12 +351,9 @@ def _grain_fields(key_column, grain):
|
|
| 397 |
else (metric["parent_asin"], PICTURE_FIELD))
|
| 398 |
where = "this ASIN or its parent ASIN" if key_column == "asin" else "this parent ASIN"
|
| 399 |
progress = _progress_fields(f"Average progress of the projects on {where}.")
|
| 400 |
-
|
| 401 |
-
if key_column == "asin" else (HISTORY_FIELD,))
|
| 402 |
-
companions = _ASIN_COMPANIONS if key_column == "asin" else ()
|
| 403 |
-
return (ids[:1] + progress[:1] + app + ids[1:] + (econ["title"],) + rollup + tuple(econ[k] for k in _ROLLUP_ECON)
|
| 404 |
+ _metric_fields(skip=("parent_asin",)) + (HEALTH_NOTE_FIELD,)
|
| 405 |
-
+ progress[1:]
|
| 406 |
|
| 407 |
|
| 408 |
ASIN_FIELDS = _grain_fields("asin", "asin")
|
|
@@ -448,7 +399,7 @@ PROJECT_COMPUTED_KEYS = tuple(f["key"] for f in PROJECT_COMPUTED_FIELDS)
|
|
| 448 |
|
| 449 |
def table_specs():
|
| 450 |
"""Every module table in creation order."""
|
| 451 |
-
|
| 452 |
{"key": TABLE_KEY, "label": TABLE_LABEL, "fields": COMPUTED_FIELDS + OWNED_FIELDS + TRACK_FIELDS,
|
| 453 |
"owned": OWNED_KEYS, "keyColumn": "sku", "template": TEMPLATE_KEY},
|
| 454 |
{"key": ASIN_KEY, "label": ASIN_LABEL, "fields": ASIN_FIELDS + COMPETITOR_FIELDS, "owned": (),
|
|
@@ -456,10 +407,6 @@ def table_specs():
|
|
| 456 |
{"key": PARENT_KEY, "label": PARENT_LABEL, "fields": PARENT_FIELDS + PARENT_COMPETITOR_FIELDS, "owned": (),
|
| 457 |
"keyColumn": "parent_asin", "template": PARENT_TEMPLATE_KEY},
|
| 458 |
)
|
| 459 |
-
for spec in specs:
|
| 460 |
-
fields = {f["key"]: f for f in spec["fields"]}
|
| 461 |
-
spec["fields"] = tuple(fields[key] for key in _project_field_order(fields))
|
| 462 |
-
return specs
|
| 463 |
|
| 464 |
|
| 465 |
# ---------------------------------------------------------------- period and threshold
|
|
@@ -1760,7 +1707,7 @@ def _sync_cursor(tenant_key, rt):
|
|
| 1760 |
else:
|
| 1761 |
import duckdb
|
| 1762 |
cur = duckdb.connect(str(path), read_only=True)
|
| 1763 |
-
if _mirror_has_economics(cur)
|
| 1764 |
return cur, cur.close
|
| 1765 |
cur.close()
|
| 1766 |
for key in [k for k in _SNAPSHOT if k[0] == tenant_key]:
|
|
@@ -1773,7 +1720,7 @@ def sync_matches(tenant_key, synced_tables, *, st=None, con=None, catalogue=None
|
|
| 1773 |
"""Refresh the current period, apply decisions, then suggest for every unlinked SKU.
|
| 1774 |
Returns the report. It raises on failure; `on_amazon_sync` is the never-raise wrapper."""
|
| 1775 |
synced = {str(t or "").strip().lower() for t in synced_tables or ()}
|
| 1776 |
-
if not synced & {"economics", "listings"
|
| 1777 |
return {"state": "skipped", "synced": sorted(synced)}
|
| 1778 |
if st is None:
|
| 1779 |
from harness import amazon_store
|
|
@@ -1785,17 +1732,6 @@ def sync_matches(tenant_key, synced_tables, *, st=None, con=None, catalogue=None
|
|
| 1785 |
con, close = _sync_cursor(tenant_key, st)
|
| 1786 |
try:
|
| 1787 |
report = {"state": "done"}
|
| 1788 |
-
if synced & {"listings", "listing_items"}:
|
| 1789 |
-
try:
|
| 1790 |
-
report["partDates"] = _part_dates.observe(st, con, day=today or _today())
|
| 1791 |
-
report["partDateCells"] = paint_asin_part_dates(st)
|
| 1792 |
-
except Exception as exc: # quality-ok: Date storage failure must not block profit matching.
|
| 1793 |
-
report['partDatesError'] = f'{type(exc).__name__}: {exc}'
|
| 1794 |
-
_log(f"{tenant_key}: listing part dates failed ({report['partDatesError']}).")
|
| 1795 |
-
if not synced & {'economics', 'listings'}:
|
| 1796 |
-
return dict(report, state='failed', reason=report['partDatesError'])
|
| 1797 |
-
if not synced & {"economics", "listings"}:
|
| 1798 |
-
return report
|
| 1799 |
try:
|
| 1800 |
refreshed = refresh(st, con, by=WRITER, today=today, tenant=tenant_key)
|
| 1801 |
report["refresh"] = {k: refreshed.get(k) for k in ("skus", "linked", "refusal")}
|
|
@@ -1871,16 +1807,12 @@ def _definition_fields(fields=None, owned_keys=None):
|
|
| 1871 |
|
| 1872 |
|
| 1873 |
def _project_field_order(keys):
|
| 1874 |
-
"""
|
| 1875 |
keys = list(keys)
|
| 1876 |
-
if
|
| 1877 |
-
|
| 1878 |
-
|
| 1879 |
-
|
| 1880 |
-
if keys[0] == "asin" else ())
|
| 1881 |
-
return ([key for key in pinned if key in keys]
|
| 1882 |
-
+ [key for key in keys if key not in pinned + companions]
|
| 1883 |
-
+ [key for key in companions if key in keys])
|
| 1884 |
|
| 1885 |
|
| 1886 |
def _format_repairs(fields, wanted):
|
|
@@ -1900,9 +1832,9 @@ def _grain_word_repairs(fields, wanted):
|
|
| 1900 |
continue
|
| 1901 |
if field["key"] == "grain" and target.get("options") and field.get("options") != target["options"]:
|
| 1902 |
out.setdefault("grain", {})["options"] = list(target["options"])
|
| 1903 |
-
if field["key"]
|
| 1904 |
or (field.get("format") or {}) != (target.get("format") or {})):
|
| 1905 |
-
out[
|
| 1906 |
return out
|
| 1907 |
|
| 1908 |
|
|
@@ -2093,9 +2025,6 @@ def write_rows(st, computed, *, table_key=TABLE_KEY, key_column="sku", label=TAB
|
|
| 2093 |
for rid, row in table_rows(st, table_key).items()
|
| 2094 |
if str(row.get(key_column) or "").strip() not in computed}
|
| 2095 |
wanted = {rid: values for rid, values in wanted.items() if values}
|
| 2096 |
-
for rid, row in table_rows(st, table_key).items():
|
| 2097 |
-
if row.get("history") != "View":
|
| 2098 |
-
wanted.setdefault(rid, {})["history"] = "View"
|
| 2099 |
if wanted:
|
| 2100 |
import core.user_tables as ut
|
| 2101 |
|
|
@@ -2639,7 +2568,7 @@ def grain_rows(rows, key, *, thin=THIN_DEFAULT, period_label="", fields=()):
|
|
| 2639 |
flat["cost_note"] = _grain_cost_note(flat, children)
|
| 2640 |
flat["period"] = period_label
|
| 2641 |
out[value] = {k: stored_cell(_FIELD_BY_KEY.get(k), flat.get(k)) for k in keys
|
| 2642 |
-
if k not in ("projects", "projects_label")
|
| 2643 |
return out
|
| 2644 |
|
| 2645 |
|
|
@@ -2871,117 +2800,8 @@ def _withhold_untied(by_grain, checks):
|
|
| 2871 |
|
| 2872 |
def _grain_period_keys(fields, key_column):
|
| 2873 |
return tuple(f["key"] for f in fields
|
| 2874 |
-
if f["key"] not in (key_column, "picture", "parent_asin", "title", "projects", "projects_label"
|
| 2875 |
-
+ COMPETITOR_KEYS
|
| 2876 |
-
|
| 2877 |
-
|
| 2878 |
-
def _listing_dates(con):
|
| 2879 |
-
"""Read the newest Amazon lastUpdatedDate per ASIN from raw listing summaries."""
|
| 2880 |
-
if "summaries" not in _columns(con, "amzn_listing_items"):
|
| 2881 |
-
return {}
|
| 2882 |
-
rows = con.execute("""SELECT json_extract_string(s.value, '$.asin'),
|
| 2883 |
-
CAST(MAX(TRY_CAST(json_extract_string(s.value, '$.lastUpdatedDate') AS TIMESTAMPTZ))
|
| 2884 |
-
AT TIME ZONE 'UTC' AS DATE)
|
| 2885 |
-
FROM amzn_listing_items l, json_each(TRY_CAST(l.summaries AS JSON)) s GROUP BY 1""").fetchall()
|
| 2886 |
-
return {asin: date.isoformat() for asin, date in rows if asin and date is not None}
|
| 2887 |
-
|
| 2888 |
-
|
| 2889 |
-
def _exposure_detail(exposure):
|
| 2890 |
-
"""Format the recorded exposure multiple with the prototype's one-decimal rounding."""
|
| 2891 |
-
if not exposure or exposure["multiple"] is None:
|
| 2892 |
-
return None
|
| 2893 |
-
multiple = Decimal.from_float(float(exposure["multiple"])).quantize(Decimal("0.1"), rounding=ROUND_HALF_UP)
|
| 2894 |
-
return f"{multiple}x {exposure['base']} share"
|
| 2895 |
-
|
| 2896 |
-
|
| 2897 |
-
def _asin_signal_cells(signal, history):
|
| 2898 |
-
"""Store scalar cells and raw monthly bars for every reported historical month."""
|
| 2899 |
-
out = {key: stored_cell(_FIELD_BY_KEY[key], signal[key]) for key in ("trend_score", "funnel_score")}
|
| 2900 |
-
out.update({key + "_reason": value for key, value in signal["reasons"].items()
|
| 2901 |
-
if key in ("trend_score", "funnel_score", "exposure")})
|
| 2902 |
-
exposure = signal["exposure"]
|
| 2903 |
-
out["exposure"] = EXPOSURES[exposure["kind"]] if exposure else None
|
| 2904 |
-
out["exposure_label"] = _exposure_detail(exposure)
|
| 2905 |
-
for kind, key, _label, _now_type in SEARCH_TRENDS:
|
| 2906 |
-
trend = signal["trends"][kind]
|
| 2907 |
-
points = {point["month"]: point for point in trend["series"]}
|
| 2908 |
-
series = [points[month] for month in history]
|
| 2909 |
-
latest = series[-1] if series else {"value": None, "reason": signal["note"] or "No search report"}
|
| 2910 |
-
now = latest["value"] if kind == "search" else signal[kind]
|
| 2911 |
-
out.update({key: None if trend["change"] is None else str(trend["change"] * 100),
|
| 2912 |
-
key + "_series": json.dumps(series, separators=(",", ":")),
|
| 2913 |
-
key + "_now": (stored_cell(_FIELD_BY_KEY[key + "_now"], now) if kind == "search"
|
| 2914 |
-
else None if now is None else str(now * 100)),
|
| 2915 |
-
key + "_reason": trend["reason"],
|
| 2916 |
-
key + "_now_reason": (signal["note"] or latest["reason"] if kind == "search"
|
| 2917 |
-
else signal["reasons"][kind])})
|
| 2918 |
-
return out
|
| 2919 |
-
|
| 2920 |
-
|
| 2921 |
-
def paint_asin_signals(st, con, *, today):
|
| 2922 |
-
"""Refresh source signals and Amazon's date for every stored ASIN, independent of the profit period."""
|
| 2923 |
-
from modules import amazon_search_signals as signals
|
| 2924 |
-
asins = [str(row["asin"]).strip() for row in table_rows(st, ASIN_KEY).values() if row.get("asin")]
|
| 2925 |
-
result = signals.signals(con, asins, today=today)
|
| 2926 |
-
dates = _listing_dates(con)
|
| 2927 |
-
parts = _part_dates.cells(st, asins)
|
| 2928 |
-
computed = {asin: dict(_asin_signal_cells(signal, result["history"]), **parts[asin], pdp_last_updated=dates.get(asin),
|
| 2929 |
-
pdp_last_updated_reason=None if asin in dates else "No listing lastUpdatedDate reported")
|
| 2930 |
-
for asin, signal in result["asins"].items()}
|
| 2931 |
-
return write_rows(st, computed, table_key=ASIN_KEY, key_column="asin", label=ASIN_LABEL, period_keys=())
|
| 2932 |
-
|
| 2933 |
-
|
| 2934 |
-
def paint_asin_part_dates(st):
|
| 2935 |
-
"""Refresh recorded part dates without profit validation, search reports or Odoo matching."""
|
| 2936 |
-
asins = [str(row['asin']).strip() for row in table_rows(st, ASIN_KEY).values() if row.get('asin')]
|
| 2937 |
-
if not asins:
|
| 2938 |
-
return {'rows': 0}
|
| 2939 |
-
return write_rows(st, _part_dates.cells(st, asins), table_key=ASIN_KEY,
|
| 2940 |
-
key_column='asin', label=ASIN_LABEL, period_keys=())
|
| 2941 |
-
|
| 2942 |
-
|
| 2943 |
-
def asin_signal_checks(st, con, *, today=None):
|
| 2944 |
-
"""Tie stored signal cells to the signal reader's independently validated raw monthly inputs."""
|
| 2945 |
-
from modules import amazon_search_signals as signals
|
| 2946 |
-
rows = {str(row["asin"]).strip(): row for row in table_rows(st, ASIN_KEY).values() if row.get("asin")}
|
| 2947 |
-
result = signals.signals(con, rows, today=today)
|
| 2948 |
-
verified = signals.validate(con, result, today=today)
|
| 2949 |
-
differences = sum(any(rows[asin].get(key) != value for key, value in _asin_signal_cells(signal, result["history"]).items())
|
| 2950 |
-
for asin, signal in result["asins"].items())
|
| 2951 |
-
return [{"check": "amazon_profit: stored ASIN search signals tie to raw monthly reports",
|
| 2952 |
-
"ok": verified["ok"] and not differences, "rows": len(rows), "differences": differences,
|
| 2953 |
-
"reason": verified["reason"]}]
|
| 2954 |
-
|
| 2955 |
-
|
| 2956 |
-
def listing_date_checks(st, con):
|
| 2957 |
-
"""Independently tie stored PDP dates to the newest raw summary timestamp per ASIN."""
|
| 2958 |
-
newest = {}
|
| 2959 |
-
available = "summaries" in _columns(con, "amzn_listing_items")
|
| 2960 |
-
if available:
|
| 2961 |
-
cursor = con.execute("SELECT summaries FROM amzn_listing_items")
|
| 2962 |
-
for batch in iter(lambda: cursor.fetchmany(2048), []):
|
| 2963 |
-
for (raw,) in batch:
|
| 2964 |
-
try:
|
| 2965 |
-
summaries = json.loads(raw) if isinstance(raw, str) else raw
|
| 2966 |
-
except (TypeError, ValueError):
|
| 2967 |
-
continue
|
| 2968 |
-
for summary in summaries if isinstance(summaries, list) else ():
|
| 2969 |
-
if not isinstance(summary, dict) or not summary.get("asin"):
|
| 2970 |
-
continue
|
| 2971 |
-
try:
|
| 2972 |
-
stamp = dt.datetime.fromisoformat(str(summary.get("lastUpdatedDate")).replace("Z", "+00:00"))
|
| 2973 |
-
except ValueError:
|
| 2974 |
-
continue
|
| 2975 |
-
stamp = stamp.replace(tzinfo=stamp.tzinfo or dt.timezone.utc).astimezone(dt.timezone.utc)
|
| 2976 |
-
asin = summary["asin"]
|
| 2977 |
-
if asin not in newest or stamp > newest[asin]:
|
| 2978 |
-
newest[asin] = stamp
|
| 2979 |
-
rows = list(table_rows(st, ASIN_KEY).values())
|
| 2980 |
-
differences = sum(row.get("pdp_last_updated") != (newest[row["asin"]].date().isoformat()
|
| 2981 |
-
if row.get("asin") in newest else None) for row in rows)
|
| 2982 |
-
return [{"check": "amazon_profit: PDP last updated ties to newest listing lastUpdatedDate per ASIN",
|
| 2983 |
-
"ok": available and not differences, "rows": len(rows), "differences": differences,
|
| 2984 |
-
"reason": None if available else "Listing summaries are unavailable"}]
|
| 2985 |
|
| 2986 |
|
| 2987 |
def _date(value):
|
|
@@ -3322,7 +3142,6 @@ def refresh(st, con, *, period=None, thin=None, by="", cost_reader=None, today=N
|
|
| 3322 |
report["parent"] = write_rows(st, by_grain["parent_asin"], table_key=PARENT_KEY,
|
| 3323 |
key_column="parent_asin", label=PARENT_LABEL,
|
| 3324 |
period_keys=_grain_period_keys(PARENT_FIELDS, "parent_asin"))
|
| 3325 |
-
report["asinSignals"] = paint_asin_signals(st, con, today=day)
|
| 3326 |
report["blankReasons"] = blank_check(stored_tables(st))
|
| 3327 |
bands = {sku: values.get("band") for sku, values in computed.items()}
|
| 3328 |
report["owned"] = apply_owned(st, bands, today=day, owner=owner_label)
|
|
@@ -3468,7 +3287,6 @@ def _mirror_has_economics(con):
|
|
| 3468 |
|
| 3469 |
|
| 3470 |
def snapshot_cursor(tenant=TENANT_ZERO, tables=(ECONOMICS, SETTLEMENTS, LISTINGS, "amzn_orders",
|
| 3471 |
-
"amzn_listing_items", "amzn_search_query_performance",
|
| 3472 |
"amzn_ads_sp_advertised_products", "amzn_ads_sb_advertised_products",
|
| 3473 |
"amzn_ads_sd_advertised_products")):
|
| 3474 |
"""An in-memory DuckDB over the stored mirror snapshot. Downloads only; writes nothing back."""
|
|
@@ -3630,10 +3448,10 @@ def stored_grain_checks(st, con):
|
|
| 3630 |
|
| 3631 |
|
| 3632 |
def validate(pre=None, month=None, con=None, *, cost_usage=None, cost_history=None, typed_rows=None,
|
| 3633 |
-
typed_values=None, typed_period=None, st=None
|
| 3634 |
"""Tie net proceeds components to independent Amazon sources within 2% over settled days.
|
| 3635 |
Receipt COGS ties to Odoo; typed COGS (`typed_rows`) is listed apart and ties to its stored values.
|
| 3636 |
-
With `st`, stored
|
| 3637 |
month = month or last_complete_month()
|
| 3638 |
bounds = period_bounds(month)
|
| 3639 |
start, end = bounds["start"], bounds["end"]
|
|
@@ -3693,8 +3511,6 @@ def validate(pre=None, month=None, con=None, *, cost_usage=None, cost_history=No
|
|
| 3693 |
checks.extend(typed_cogs_checks(con, typed_rows, typed_values, cost_usage, typed_period or bounds))
|
| 3694 |
if st is not None:
|
| 3695 |
checks.extend(stored_grain_checks(st, con))
|
| 3696 |
-
checks.extend(listing_date_checks(st, con))
|
| 3697 |
-
checks.extend(asin_signal_checks(st, con, today=today))
|
| 3698 |
return checks
|
| 3699 |
|
| 3700 |
|
|
|
|
| 31 |
import shutil
|
| 32 |
import tempfile
|
| 33 |
import threading
|
|
|
|
| 34 |
from pathlib import Path
|
| 35 |
|
| 36 |
from modules import amazon_sku_metrics as M
|
|
|
|
| 37 |
import core.user_tables as _registry
|
| 38 |
from core import module_tables
|
| 39 |
from core import amazon_rivals as _rivals
|
|
|
|
| 119 |
return out
|
| 120 |
|
| 121 |
|
| 122 |
+
def _progress_fields(description):
|
| 123 |
"""Project progress and its completed-count label, written by the Projects doors."""
|
| 124 |
+
return (_field("projects", "Projects", "pct", description, computed=True,
|
| 125 |
+
format={"display": "progress", "labelKey": "projects_label", "emptyLabel": "No projects"}),
|
| 126 |
+
_field("projects_label", "Projects completed", "text", "Completed projects of the counted ones.",
|
|
|
|
| 127 |
computed=True))
|
| 128 |
|
| 129 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 130 |
def _metric_fields(skip=()):
|
| 131 |
"""The sibling metric module's columns, as computed fields of this module."""
|
| 132 |
return tuple(_field(key, label, ftype, description, computed=True, **dict(extra or {}))
|
|
|
|
| 180 |
for k, label, t, d, x in _rivals.PARENT_COLUMNS)
|
| 181 |
COMPETITOR_KEYS = _rivals.ROW_KEYS + _rivals.PARENT_KEYS
|
| 182 |
_SKU_PROJECT_FIELDS = _progress_fields("Average progress of the projects on this SKU, its ASIN and its parent ASIN.")
|
| 183 |
+
COMPUTED_FIELDS = (_ECON_FIELDS[:1] + _SKU_PROJECT_FIELDS[:1] + (PICTURE_FIELD,) + _ECON_FIELDS[1:] + _metric_fields()
|
| 184 |
+ (HEALTH_NOTE_FIELD,) + _SKU_PROJECT_FIELDS[1:]
|
| 185 |
+ COMPETITOR_FIELDS)
|
| 186 |
#: Owned columns: people edit them and a refresh never writes them.
|
|
|
|
| 332 |
if source and row.get("cogs_per_unit") is None:
|
| 333 |
return source if source.endswith(".") else source + "."
|
| 334 |
return NO_COST_NOTE
|
| 335 |
+
_IDENTITY_KEYS = ("sku", "picture", "asin", "parent_asin", "title", "projects", "projects_label")
|
| 336 |
_PERIOD_KEYS = tuple(k for k in COMPUTED_KEYS if k not in _IDENTITY_KEYS + COMPETITOR_KEYS)
|
| 337 |
|
| 338 |
#: The profit block a rollup table carries; `_ECON_ADDITIVE` sums, margin and band recompute.
|
|
|
|
| 351 |
else (metric["parent_asin"], PICTURE_FIELD))
|
| 352 |
where = "this ASIN or its parent ASIN" if key_column == "asin" else "this parent ASIN"
|
| 353 |
progress = _progress_fields(f"Average progress of the projects on {where}.")
|
| 354 |
+
return (ids[:1] + progress[:1] + ids[1:] + (econ["title"],) + rollup + tuple(econ[k] for k in _ROLLUP_ECON)
|
|
|
|
|
|
|
|
|
|
| 355 |
+ _metric_fields(skip=("parent_asin",)) + (HEALTH_NOTE_FIELD,)
|
| 356 |
+
+ progress[1:])
|
| 357 |
|
| 358 |
|
| 359 |
ASIN_FIELDS = _grain_fields("asin", "asin")
|
|
|
|
| 399 |
|
| 400 |
def table_specs():
|
| 401 |
"""Every module table in creation order."""
|
| 402 |
+
return (
|
| 403 |
{"key": TABLE_KEY, "label": TABLE_LABEL, "fields": COMPUTED_FIELDS + OWNED_FIELDS + TRACK_FIELDS,
|
| 404 |
"owned": OWNED_KEYS, "keyColumn": "sku", "template": TEMPLATE_KEY},
|
| 405 |
{"key": ASIN_KEY, "label": ASIN_LABEL, "fields": ASIN_FIELDS + COMPETITOR_FIELDS, "owned": (),
|
|
|
|
| 407 |
{"key": PARENT_KEY, "label": PARENT_LABEL, "fields": PARENT_FIELDS + PARENT_COMPETITOR_FIELDS, "owned": (),
|
| 408 |
"keyColumn": "parent_asin", "template": PARENT_TEMPLATE_KEY},
|
| 409 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 410 |
|
| 411 |
|
| 412 |
# ---------------------------------------------------------------- period and threshold
|
|
|
|
| 1707 |
else:
|
| 1708 |
import duckdb
|
| 1709 |
cur = duckdb.connect(str(path), read_only=True)
|
| 1710 |
+
if _mirror_has_economics(cur):
|
| 1711 |
return cur, cur.close
|
| 1712 |
cur.close()
|
| 1713 |
for key in [k for k in _SNAPSHOT if k[0] == tenant_key]:
|
|
|
|
| 1720 |
"""Refresh the current period, apply decisions, then suggest for every unlinked SKU.
|
| 1721 |
Returns the report. It raises on failure; `on_amazon_sync` is the never-raise wrapper."""
|
| 1722 |
synced = {str(t or "").strip().lower() for t in synced_tables or ()}
|
| 1723 |
+
if not synced & {"economics", "listings"}:
|
| 1724 |
return {"state": "skipped", "synced": sorted(synced)}
|
| 1725 |
if st is None:
|
| 1726 |
from harness import amazon_store
|
|
|
|
| 1732 |
con, close = _sync_cursor(tenant_key, st)
|
| 1733 |
try:
|
| 1734 |
report = {"state": "done"}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1735 |
try:
|
| 1736 |
refreshed = refresh(st, con, by=WRITER, today=today, tenant=tenant_key)
|
| 1737 |
report["refresh"] = {k: refreshed.get(k) for k in ("skus", "linked", "refusal")}
|
|
|
|
| 1807 |
|
| 1808 |
|
| 1809 |
def _project_field_order(keys):
|
| 1810 |
+
"""Place Projects beside a product identity in schema defaults, never saved user layouts."""
|
| 1811 |
keys = list(keys)
|
| 1812 |
+
if keys and keys[0] in ("sku", "asin", "parent_asin") and "projects" in keys:
|
| 1813 |
+
keys.remove("projects")
|
| 1814 |
+
keys.insert(1, "projects")
|
| 1815 |
+
return keys
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1816 |
|
| 1817 |
|
| 1818 |
def _format_repairs(fields, wanted):
|
|
|
|
| 1832 |
continue
|
| 1833 |
if field["key"] == "grain" and target.get("options") and field.get("options") != target["options"]:
|
| 1834 |
out.setdefault("grain", {})["options"] = list(target["options"])
|
| 1835 |
+
if field["key"] == "projects" and (field.get("type") != target["type"] or "link" in field
|
| 1836 |
or (field.get("format") or {}) != (target.get("format") or {})):
|
| 1837 |
+
out["projects"] = {"type": target["type"], "format": dict(target.get("format") or {}), "link": None}
|
| 1838 |
return out
|
| 1839 |
|
| 1840 |
|
|
|
|
| 2025 |
for rid, row in table_rows(st, table_key).items()
|
| 2026 |
if str(row.get(key_column) or "").strip() not in computed}
|
| 2027 |
wanted = {rid: values for rid, values in wanted.items() if values}
|
|
|
|
|
|
|
|
|
|
| 2028 |
if wanted:
|
| 2029 |
import core.user_tables as ut
|
| 2030 |
|
|
|
|
| 2568 |
flat["cost_note"] = _grain_cost_note(flat, children)
|
| 2569 |
flat["period"] = period_label
|
| 2570 |
out[value] = {k: stored_cell(_FIELD_BY_KEY.get(k), flat.get(k)) for k in keys
|
| 2571 |
+
if k not in ("projects", "projects_label")}
|
| 2572 |
return out
|
| 2573 |
|
| 2574 |
|
|
|
|
| 2800 |
|
| 2801 |
def _grain_period_keys(fields, key_column):
|
| 2802 |
return tuple(f["key"] for f in fields
|
| 2803 |
+
if f["key"] not in (key_column, "picture", "parent_asin", "title", "projects", "projects_label")
|
| 2804 |
+
+ COMPETITOR_KEYS)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2805 |
|
| 2806 |
|
| 2807 |
def _date(value):
|
|
|
|
| 3142 |
report["parent"] = write_rows(st, by_grain["parent_asin"], table_key=PARENT_KEY,
|
| 3143 |
key_column="parent_asin", label=PARENT_LABEL,
|
| 3144 |
period_keys=_grain_period_keys(PARENT_FIELDS, "parent_asin"))
|
|
|
|
| 3145 |
report["blankReasons"] = blank_check(stored_tables(st))
|
| 3146 |
bands = {sku: values.get("band") for sku, values in computed.items()}
|
| 3147 |
report["owned"] = apply_owned(st, bands, today=day, owner=owner_label)
|
|
|
|
| 3287 |
|
| 3288 |
|
| 3289 |
def snapshot_cursor(tenant=TENANT_ZERO, tables=(ECONOMICS, SETTLEMENTS, LISTINGS, "amzn_orders",
|
|
|
|
| 3290 |
"amzn_ads_sp_advertised_products", "amzn_ads_sb_advertised_products",
|
| 3291 |
"amzn_ads_sd_advertised_products")):
|
| 3292 |
"""An in-memory DuckDB over the stored mirror snapshot. Downloads only; writes nothing back."""
|
|
|
|
| 3448 |
|
| 3449 |
|
| 3450 |
def validate(pre=None, month=None, con=None, *, cost_usage=None, cost_history=None, typed_rows=None,
|
| 3451 |
+
typed_values=None, typed_period=None, st=None):
|
| 3452 |
"""Tie net proceeds components to independent Amazon sources within 2% over settled days.
|
| 3453 |
Receipt COGS ties to Odoo; typed COGS (`typed_rows`) is listed apart and ties to its stored values.
|
| 3454 |
+
With `st`, the stored ASIN and parent figures tie per row to the mirror and every blank says why."""
|
| 3455 |
month = month or last_complete_month()
|
| 3456 |
bounds = period_bounds(month)
|
| 3457 |
start, end = bounds["start"], bounds["end"]
|
|
|
|
| 3511 |
checks.extend(typed_cogs_checks(con, typed_rows, typed_values, cost_usage, typed_period or bounds))
|
| 3512 |
if st is not None:
|
| 3513 |
checks.extend(stored_grain_checks(st, con))
|
|
|
|
|
|
|
| 3514 |
return checks
|
| 3515 |
|
| 3516 |
|
platform/modules/amazon_profit_account.py
CHANGED
|
@@ -555,124 +555,6 @@ def validate_pace(con, bounds, pace, elapsed, month_days, reason):
|
|
| 555 |
"checks": checks, "sourceTables": [ECONOMICS, "_sync_state"]}
|
| 556 |
|
| 557 |
|
| 558 |
-
class _SourceRows:
|
| 559 |
-
def __init__(self, rows):
|
| 560 |
-
self.rows = rows
|
| 561 |
-
|
| 562 |
-
def fetchone(self):
|
| 563 |
-
return self.rows[0] if self.rows else None
|
| 564 |
-
|
| 565 |
-
def fetchall(self):
|
| 566 |
-
return list(self.rows)
|
| 567 |
-
|
| 568 |
-
|
| 569 |
-
def _settlement_source(sql):
|
| 570 |
-
"""Parse distinct settlement labels and referenced order dates without losing source lines or null keys."""
|
| 571 |
-
expressions = {"posted": ("posted_date", f"CAST({profit._moment_sql('s.posted_date')} AS DATE)")}
|
| 572 |
-
for name, field in (("description", "amount_description"), ("kind", "amount_type"),
|
| 573 |
-
("transaction", "transaction_type")):
|
| 574 |
-
expressions[name] = field, f"lower(regexp_replace(COALESCE({field}, ''), '[^a-zA-Z0-9]', '', 'g'))"
|
| 575 |
-
fields = [field for field, _expression in expressions.values()]
|
| 576 |
-
projection = ", ".join(expression.replace("s.posted_date", "posted_date") + " AS " + name
|
| 577 |
-
for name, (_field, expression) in expressions.items())
|
| 578 |
-
keys = ", ".join(f"{field} AS source_{field}" for field in fields)
|
| 579 |
-
labels = (f"labels AS MATERIALIZED (SELECT {keys}, {projection} "
|
| 580 |
-
f"FROM (SELECT DISTINCT {', '.join(fields)} FROM {profit.SETTLEMENTS})), ")
|
| 581 |
-
for name, (_field, expression) in expressions.items():
|
| 582 |
-
sql = sql.replace(expression + " AS " + name, "label." + name + " AS " + name)
|
| 583 |
-
join = " AND ".join(f"s.{field} IS NOT DISTINCT FROM label.source_{field}" for field in fields)
|
| 584 |
-
sql = sql.replace(f"FROM {profit.SETTLEMENTS} s JOIN streams",
|
| 585 |
-
f"FROM {profit.SETTLEMENTS} s JOIN labels label ON {join} JOIN streams")
|
| 586 |
-
sql = sql.replace("FROM amzn_orders GROUP BY amazon_order_id",
|
| 587 |
-
f"FROM amzn_orders WHERE amazon_order_id IN (SELECT trim(order_id) "
|
| 588 |
-
f"FROM {profit.SETTLEMENTS}) GROUP BY amazon_order_id")
|
| 589 |
-
return sql.replace("WITH orders AS (", "WITH " + labels + "orders AS (", 1)
|
| 590 |
-
|
| 591 |
-
|
| 592 |
-
class _ValidationSources:
|
| 593 |
-
"""Share settlement source preparation across this request's independent window sums."""
|
| 594 |
-
|
| 595 |
-
def __init__(self, con, bounds):
|
| 596 |
-
self.con, self.results, self.spans = con, {}, {}
|
| 597 |
-
self.bounds = [span for bound in bounds for span in (bound, *_comparisons(bound).values())]
|
| 598 |
-
|
| 599 |
-
def _spans(self, currency=None):
|
| 600 |
-
if currency in self.spans:
|
| 601 |
-
return self.spans[currency]
|
| 602 |
-
spans = {(bound["start"], bound["end"]) for bound in self.bounds}
|
| 603 |
-
if currency is not None:
|
| 604 |
-
for start, end in tuple(spans):
|
| 605 |
-
covered = profit.settlement_coverage(self, start, end, currency)
|
| 606 |
-
spans.add((covered["start"] or start, covered["end"] or end))
|
| 607 |
-
self.spans[currency] = sorted(spans)
|
| 608 |
-
return self.spans[currency]
|
| 609 |
-
|
| 610 |
-
def _components(self, sql, params):
|
| 611 |
-
marker = "SELECT component, COALESCE(SUM(currency_amount), 0)"
|
| 612 |
-
prefix, select = sql.split(marker, 1)
|
| 613 |
-
aggregates, select = (marker + select).split("FROM dated WHERE", 1)
|
| 614 |
-
aggregates = aggregates.removeprefix("SELECT component, ")
|
| 615 |
-
# The grouping keys retain every input to the original window predicate and its bad-row counts.
|
| 616 |
-
prefix += (" , prepared(component, basis, posted, dates, bad_dates, amount, source_rows, "
|
| 617 |
-
"bad_amounts, unmatched_orders) AS (SELECT component, basis, posted, dates, bad_dates, "
|
| 618 |
-
+ aggregates + " FROM dated GROUP BY component, basis, posted, dates, bad_dates) ")
|
| 619 |
-
spans = self._spans(params[0])
|
| 620 |
-
for bound in ("lo", "hi", "lo", "hi"):
|
| 621 |
-
select = select.replace("CAST(? AS DATE)", "span." + bound, 1)
|
| 622 |
-
ranges = ", ".join(f"({index}, CAST(? AS DATE), CAST(? AS DATE))" for index in range(len(spans)))
|
| 623 |
-
select = f"FROM prepared JOIN (VALUES {ranges}) AS span(id, lo, hi) ON" + select
|
| 624 |
-
select = select.replace("GROUP BY component ORDER BY component",
|
| 625 |
-
"GROUP BY span.id, component ORDER BY span.id, component")
|
| 626 |
-
bindings = list(params[:2]) + [value for span in spans for value in span]
|
| 627 |
-
rows = self.con.execute(_settlement_source(prefix) + "SELECT span.id, component, "
|
| 628 |
-
"COALESCE(SUM(amount), 0), SUM(source_rows), SUM(bad_amounts), SUM(unmatched_orders) "
|
| 629 |
-
+ select, bindings).fetchall()
|
| 630 |
-
for index, span in enumerate(spans):
|
| 631 |
-
self.results[(sql, tuple(params[:2]) + span * 2)] = [row[1:] for row in rows if row[0] == index]
|
| 632 |
-
|
| 633 |
-
def _unscoped(self, sql):
|
| 634 |
-
marker = "AND (CAST("
|
| 635 |
-
source, _date_filter = sql.split(marker, 1)
|
| 636 |
-
source = source.replace("SELECT count(*)", f"SELECT CAST({profit._moment_sql('posted_date')} AS DATE) AS posted", 1)
|
| 637 |
-
spans = self._spans(profit.economics_currency(self.con))
|
| 638 |
-
sums = ["count(*) FILTER (WHERE posted BETWEEN CAST(? AS DATE) AND CAST(? AS DATE) OR posted IS NULL)"
|
| 639 |
-
for _span in spans]
|
| 640 |
-
counts = self.con.execute("SELECT " + ", ".join(sums) + " FROM (" + source + ")",
|
| 641 |
-
[value for span in spans for value in span]).fetchone()
|
| 642 |
-
for index, count in enumerate(counts):
|
| 643 |
-
self.results[(sql, spans[index])] = [(count,)]
|
| 644 |
-
|
| 645 |
-
def execute(self, sql, params=()):
|
| 646 |
-
params = tuple(params or ())
|
| 647 |
-
key = sql, params
|
| 648 |
-
if key in self.results:
|
| 649 |
-
return _SourceRows(self.results[key])
|
| 650 |
-
if sql == "SELECT column_name FROM information_schema.columns WHERE table_name = ?":
|
| 651 |
-
self.results[key] = self.con.execute(sql, params).fetchall()
|
| 652 |
-
return _SourceRows(self.results[key])
|
| 653 |
-
components = (sql.startswith("WITH orders AS (") and len(params) == 6
|
| 654 |
-
and sql.endswith(" GROUP BY component ORDER BY component"))
|
| 655 |
-
unscoped = (sql.startswith(f"SELECT count(*) FROM {profit.SETTLEMENTS} s\n")
|
| 656 |
-
and "WHERE h.settlement_id IS NULL" in sql and len(params) == 2)
|
| 657 |
-
if components:
|
| 658 |
-
self._components(sql, params)
|
| 659 |
-
elif unscoped:
|
| 660 |
-
self._unscoped(sql)
|
| 661 |
-
if key in self.results:
|
| 662 |
-
return _SourceRows(self.results[key])
|
| 663 |
-
return self.con.execute(sql, params)
|
| 664 |
-
|
| 665 |
-
|
| 666 |
-
def _account_bounds(chosen, today):
|
| 667 |
-
bounds = [chosen, _bounds(today.replace(day=1), today - dt.timedelta(days=1), "mtd")]
|
| 668 |
-
current = today.replace(day=1)
|
| 669 |
-
for _ in range(2):
|
| 670 |
-
last = current - dt.timedelta(days=1)
|
| 671 |
-
current = last.replace(day=1)
|
| 672 |
-
bounds.append(_bounds(current, last, current.strftime("%Y-%m")))
|
| 673 |
-
return bounds
|
| 674 |
-
|
| 675 |
-
|
| 676 |
def account_view(con, period=None, *, stored=None, settings=None, today=None):
|
| 677 |
"""Return selected, comparison and monthly account windows without refreshing stored inputs."""
|
| 678 |
today = today or profit._today()
|
|
@@ -692,8 +574,6 @@ def account_view(con, period=None, *, stored=None, settings=None, today=None):
|
|
| 692 |
"TRY_CAST(left(CAST(start_date AS VARCHAR),10) AS DATE) IS NULL").fetchone()[0]:
|
| 693 |
raise profit.SourceUnavailable("Economics dates unavailable. Resync Amazon economics.")
|
| 694 |
stored, settings = stored or {}, settings or {}
|
| 695 |
-
periods = _account_bounds(chosen, today)
|
| 696 |
-
con = _ValidationSources(con, periods)
|
| 697 |
cache = {}
|
| 698 |
def window(bounds):
|
| 699 |
key = bounds["start"], bounds["end"]
|
|
@@ -702,8 +582,13 @@ def account_view(con, period=None, *, stored=None, settings=None, today=None):
|
|
| 702 |
return {**cache[key], "period": bounds}
|
| 703 |
def compared(bounds):
|
| 704 |
return {**window(bounds), "comparisons": {key: window(span) for key, span in _comparisons(bounds).items()}}
|
| 705 |
-
|
| 706 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 707 |
pace = {}
|
| 708 |
elapsed = mtd["period"]["days"]
|
| 709 |
month_days = calendar.monthrange(today.year, today.month)[1]
|
|
|
|
| 555 |
"checks": checks, "sourceTables": [ECONOMICS, "_sync_state"]}
|
| 556 |
|
| 557 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 558 |
def account_view(con, period=None, *, stored=None, settings=None, today=None):
|
| 559 |
"""Return selected, comparison and monthly account windows without refreshing stored inputs."""
|
| 560 |
today = today or profit._today()
|
|
|
|
| 574 |
"TRY_CAST(left(CAST(start_date AS VARCHAR),10) AS DATE) IS NULL").fetchone()[0]:
|
| 575 |
raise profit.SourceUnavailable("Economics dates unavailable. Resync Amazon economics.")
|
| 576 |
stored, settings = stored or {}, settings or {}
|
|
|
|
|
|
|
| 577 |
cache = {}
|
| 578 |
def window(bounds):
|
| 579 |
key = bounds["start"], bounds["end"]
|
|
|
|
| 582 |
return {**cache[key], "period": bounds}
|
| 583 |
def compared(bounds):
|
| 584 |
return {**window(bounds), "comparisons": {key: window(span) for key, span in _comparisons(bounds).items()}}
|
| 585 |
+
current = today.replace(day=1)
|
| 586 |
+
mtd = compared(_bounds(current, today - dt.timedelta(days=1), "mtd"))
|
| 587 |
+
months = []
|
| 588 |
+
for _ in range(2):
|
| 589 |
+
last = current - dt.timedelta(days=1)
|
| 590 |
+
current = last.replace(day=1)
|
| 591 |
+
months.append(compared(_bounds(current, last, current.strftime("%Y-%m"))))
|
| 592 |
pace = {}
|
| 593 |
elapsed = mtd["period"]["days"]
|
| 594 |
month_days = calendar.monthrange(today.year, today.month)[1]
|
platform/modules/competitor_watch.py
CHANGED
|
@@ -300,7 +300,7 @@ class _TermIndex(_DocumentIndex):
|
|
| 300 |
self.sql("SELECT (value ->> 'activeTotal')::bigint FROM {tbl} WHERE key = %s"))
|
| 301 |
total_params = ([self.slug, *([display_term(search)] if search else [])]
|
| 302 |
if search or tracked else [self.key(KEY_TERM_INDEX)])
|
| 303 |
-
|
| 304 |
"bounds AS (SELECT total, CASE WHEN %s >= total THEN "
|
| 305 |
"greatest(0, floor((total - 1)::numeric / %s)::bigint * %s) ELSE %s END AS skip FROM totals) "
|
| 306 |
"SELECT total, skip, EXISTS(SELECT 1 FROM {terms} WHERE tenant = %s AND term = %s AND " + membership + "), "
|
|
@@ -308,12 +308,6 @@ class _TermIndex(_DocumentIndex):
|
|
| 308 |
+ ("AND strpos(lower(label), %s) > 0 " if search else "") +
|
| 309 |
'ORDER BY ' + order + ' LIMIT %s OFFSET (SELECT skip FROM bounds)) '
|
| 310 |
"FROM bounds")
|
| 311 |
-
if isinstance(total, str):
|
| 312 |
-
statement = self.term_sql("WITH totals(total) AS (" + total + tail)
|
| 313 |
-
else:
|
| 314 |
-
# PostgreSQL's `self.sql` returns a psycopg Composed: compose it, never concatenate it as text.
|
| 315 |
-
from psycopg import sql as psql
|
| 316 |
-
statement = psql.SQL("WITH totals(total) AS (") + total + psql.SQL(self.term_sql(tail))
|
| 317 |
params = [*total_params, offset, limit, limit, offset, self.slug, selected, self.slug,
|
| 318 |
*([display_term(search)] if search else []), limit]
|
| 319 |
with self.connection() as con:
|
|
@@ -469,25 +463,15 @@ def _ba_columns(cur):
|
|
| 469 |
|
| 470 |
|
| 471 |
def _sqp(cur, term=None):
|
| 472 |
-
"""
|
| 473 |
need = {"search_query_data_search_query", "asin"}
|
| 474 |
cols = _columns(cur, SQP_TABLE) if cur is not None else set()
|
| 475 |
if not need <= cols:
|
| 476 |
return []
|
| 477 |
-
if "_report_period" in cols:
|
| 478 |
-
period = '"_report_period" = \'MONTH\''
|
| 479 |
-
elif {"start_date", "end_date"} <= cols:
|
| 480 |
-
start = 'TRY_CAST("start_date" AS DATE)'
|
| 481 |
-
end_date = 'TRY_CAST("end_date" AS DATE)'
|
| 482 |
-
period = f'day({start}) = 1 AND {end_date} = last_day({start})'
|
| 483 |
-
else:
|
| 484 |
-
return []
|
| 485 |
imp = ('TRY_CAST("impression_data_asin_impression_count" AS DOUBLE)'
|
| 486 |
if "impression_data_asin_impression_count" in cols else "0")
|
| 487 |
end = 'CAST("end_date" AS VARCHAR)' if "end_date" in cols else "''"
|
| 488 |
-
where = f
|
| 489 |
-
if term is not None:
|
| 490 |
-
where += f' AND {_term_sql("search_query_data_search_query")} = ?'
|
| 491 |
rows = cur.execute(
|
| 492 |
f'SELECT "search_query_data_search_query", "asin", {imp}, {end} FROM {SQP_TABLE}{where}',
|
| 493 |
[term] if term is not None else []).fetchall()
|
|
|
|
| 300 |
self.sql("SELECT (value ->> 'activeTotal')::bigint FROM {tbl} WHERE key = %s"))
|
| 301 |
total_params = ([self.slug, *([display_term(search)] if search else [])]
|
| 302 |
if search or tracked else [self.key(KEY_TERM_INDEX)])
|
| 303 |
+
statement = self.term_sql("WITH totals(total) AS (" + total + "), "
|
| 304 |
"bounds AS (SELECT total, CASE WHEN %s >= total THEN "
|
| 305 |
"greatest(0, floor((total - 1)::numeric / %s)::bigint * %s) ELSE %s END AS skip FROM totals) "
|
| 306 |
"SELECT total, skip, EXISTS(SELECT 1 FROM {terms} WHERE tenant = %s AND term = %s AND " + membership + "), "
|
|
|
|
| 308 |
+ ("AND strpos(lower(label), %s) > 0 " if search else "") +
|
| 309 |
'ORDER BY ' + order + ' LIMIT %s OFFSET (SELECT skip FROM bounds)) '
|
| 310 |
"FROM bounds")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 311 |
params = [*total_params, offset, limit, limit, offset, self.slug, selected, self.slug,
|
| 312 |
*([display_term(search)] if search else []), limit]
|
| 313 |
with self.connection() as con:
|
|
|
|
| 463 |
|
| 464 |
|
| 465 |
def _sqp(cur, term=None):
|
| 466 |
+
"""SQP rows as (term, asin, impressions, end_date). SQP reports our ASINs only."""
|
| 467 |
need = {"search_query_data_search_query", "asin"}
|
| 468 |
cols = _columns(cur, SQP_TABLE) if cur is not None else set()
|
| 469 |
if not need <= cols:
|
| 470 |
return []
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 471 |
imp = ('TRY_CAST("impression_data_asin_impression_count" AS DOUBLE)'
|
| 472 |
if "impression_data_asin_impression_count" in cols else "0")
|
| 473 |
end = 'CAST("end_date" AS VARCHAR)' if "end_date" in cols else "''"
|
| 474 |
+
where = f' WHERE {_term_sql("search_query_data_search_query")} = ?' if term is not None else ""
|
|
|
|
|
|
|
| 475 |
rows = cur.execute(
|
| 476 |
f'SELECT "search_query_data_search_query", "asin", {imp}, {end} FROM {SQP_TABLE}{where}',
|
| 477 |
[term] if term is not None else []).fetchall()
|
requirements.txt
CHANGED
|
@@ -25,7 +25,6 @@ duckdb==1.5.4
|
|
| 25 |
# duckdb's pandas bridge imports pytz for timezone values; pandas 3 no longer installs it.
|
| 26 |
pytz>=2024.1
|
| 27 |
pyyaml>=6.0
|
| 28 |
-
jsonschema==4.26.0
|
| 29 |
pillow>=10.0
|
| 30 |
beautifulsoup4>=4.12
|
| 31 |
lxml>=5.0
|
|
|
|
| 25 |
# duckdb's pandas bridge imports pytz for timezone values; pandas 3 no longer installs it.
|
| 26 |
pytz>=2024.1
|
| 27 |
pyyaml>=6.0
|
|
|
|
| 28 |
pillow>=10.0
|
| 29 |
beautifulsoup4>=4.12
|
| 30 |
lxml>=5.0
|
web/package-lock.json
CHANGED
|
@@ -25,7 +25,6 @@
|
|
| 25 |
"@types/react": "^18.3.12",
|
| 26 |
"@types/react-dom": "^18.3.1",
|
| 27 |
"@vitejs/plugin-react": "^6.0.3",
|
| 28 |
-
"esbuild": "^0.27.0",
|
| 29 |
"openapi-typescript": "^7.13.0",
|
| 30 |
"oxlint": "^1.71.0",
|
| 31 |
"typescript": "~6.0.2",
|
|
@@ -390,448 +389,6 @@
|
|
| 390 |
"integrity": "sha512-30FAj7/EoJ5mwVPOWhAyCX+FPfMDrVecJAM+Iw9NRoSl4BBAQeqj4cApHHUXOVvIPgLVDsCFoz/hGD+5QQD1GQ==",
|
| 391 |
"license": "MIT"
|
| 392 |
},
|
| 393 |
-
"node_modules/@esbuild/aix-ppc64": {
|
| 394 |
-
"version": "0.27.7",
|
| 395 |
-
"resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.27.7.tgz",
|
| 396 |
-
"integrity": "sha512-EKX3Qwmhz1eMdEJokhALr0YiD0lhQNwDqkPYyPhiSwKrh7/4KRjQc04sZ8db+5DVVnZ1LmbNDI1uAMPEUBnQPg==",
|
| 397 |
-
"cpu": [
|
| 398 |
-
"ppc64"
|
| 399 |
-
],
|
| 400 |
-
"dev": true,
|
| 401 |
-
"license": "MIT",
|
| 402 |
-
"optional": true,
|
| 403 |
-
"os": [
|
| 404 |
-
"aix"
|
| 405 |
-
],
|
| 406 |
-
"engines": {
|
| 407 |
-
"node": ">=18"
|
| 408 |
-
}
|
| 409 |
-
},
|
| 410 |
-
"node_modules/@esbuild/android-arm": {
|
| 411 |
-
"version": "0.27.7",
|
| 412 |
-
"resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.27.7.tgz",
|
| 413 |
-
"integrity": "sha512-jbPXvB4Yj2yBV7HUfE2KHe4GJX51QplCN1pGbYjvsyCZbQmies29EoJbkEc+vYuU5o45AfQn37vZlyXy4YJ8RQ==",
|
| 414 |
-
"cpu": [
|
| 415 |
-
"arm"
|
| 416 |
-
],
|
| 417 |
-
"dev": true,
|
| 418 |
-
"license": "MIT",
|
| 419 |
-
"optional": true,
|
| 420 |
-
"os": [
|
| 421 |
-
"android"
|
| 422 |
-
],
|
| 423 |
-
"engines": {
|
| 424 |
-
"node": ">=18"
|
| 425 |
-
}
|
| 426 |
-
},
|
| 427 |
-
"node_modules/@esbuild/android-arm64": {
|
| 428 |
-
"version": "0.27.7",
|
| 429 |
-
"resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.27.7.tgz",
|
| 430 |
-
"integrity": "sha512-62dPZHpIXzvChfvfLJow3q5dDtiNMkwiRzPylSCfriLvZeq0a1bWChrGx/BbUbPwOrsWKMn8idSllklzBy+dgQ==",
|
| 431 |
-
"cpu": [
|
| 432 |
-
"arm64"
|
| 433 |
-
],
|
| 434 |
-
"dev": true,
|
| 435 |
-
"license": "MIT",
|
| 436 |
-
"optional": true,
|
| 437 |
-
"os": [
|
| 438 |
-
"android"
|
| 439 |
-
],
|
| 440 |
-
"engines": {
|
| 441 |
-
"node": ">=18"
|
| 442 |
-
}
|
| 443 |
-
},
|
| 444 |
-
"node_modules/@esbuild/android-x64": {
|
| 445 |
-
"version": "0.27.7",
|
| 446 |
-
"resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.27.7.tgz",
|
| 447 |
-
"integrity": "sha512-x5VpMODneVDb70PYV2VQOmIUUiBtY3D3mPBG8NxVk5CogneYhkR7MmM3yR/uMdITLrC1ml/NV1rj4bMJuy9MCg==",
|
| 448 |
-
"cpu": [
|
| 449 |
-
"x64"
|
| 450 |
-
],
|
| 451 |
-
"dev": true,
|
| 452 |
-
"license": "MIT",
|
| 453 |
-
"optional": true,
|
| 454 |
-
"os": [
|
| 455 |
-
"android"
|
| 456 |
-
],
|
| 457 |
-
"engines": {
|
| 458 |
-
"node": ">=18"
|
| 459 |
-
}
|
| 460 |
-
},
|
| 461 |
-
"node_modules/@esbuild/darwin-arm64": {
|
| 462 |
-
"version": "0.27.7",
|
| 463 |
-
"resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.27.7.tgz",
|
| 464 |
-
"integrity": "sha512-5lckdqeuBPlKUwvoCXIgI2D9/ABmPq3Rdp7IfL70393YgaASt7tbju3Ac+ePVi3KDH6N2RqePfHnXkaDtY9fkw==",
|
| 465 |
-
"cpu": [
|
| 466 |
-
"arm64"
|
| 467 |
-
],
|
| 468 |
-
"dev": true,
|
| 469 |
-
"license": "MIT",
|
| 470 |
-
"optional": true,
|
| 471 |
-
"os": [
|
| 472 |
-
"darwin"
|
| 473 |
-
],
|
| 474 |
-
"engines": {
|
| 475 |
-
"node": ">=18"
|
| 476 |
-
}
|
| 477 |
-
},
|
| 478 |
-
"node_modules/@esbuild/darwin-x64": {
|
| 479 |
-
"version": "0.27.7",
|
| 480 |
-
"resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.27.7.tgz",
|
| 481 |
-
"integrity": "sha512-rYnXrKcXuT7Z+WL5K980jVFdvVKhCHhUwid+dDYQpH+qu+TefcomiMAJpIiC2EM3Rjtq0sO3StMV/+3w3MyyqQ==",
|
| 482 |
-
"cpu": [
|
| 483 |
-
"x64"
|
| 484 |
-
],
|
| 485 |
-
"dev": true,
|
| 486 |
-
"license": "MIT",
|
| 487 |
-
"optional": true,
|
| 488 |
-
"os": [
|
| 489 |
-
"darwin"
|
| 490 |
-
],
|
| 491 |
-
"engines": {
|
| 492 |
-
"node": ">=18"
|
| 493 |
-
}
|
| 494 |
-
},
|
| 495 |
-
"node_modules/@esbuild/freebsd-arm64": {
|
| 496 |
-
"version": "0.27.7",
|
| 497 |
-
"resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.27.7.tgz",
|
| 498 |
-
"integrity": "sha512-B48PqeCsEgOtzME2GbNM2roU29AMTuOIN91dsMO30t+Ydis3z/3Ngoj5hhnsOSSwNzS+6JppqWsuhTp6E82l2w==",
|
| 499 |
-
"cpu": [
|
| 500 |
-
"arm64"
|
| 501 |
-
],
|
| 502 |
-
"dev": true,
|
| 503 |
-
"license": "MIT",
|
| 504 |
-
"optional": true,
|
| 505 |
-
"os": [
|
| 506 |
-
"freebsd"
|
| 507 |
-
],
|
| 508 |
-
"engines": {
|
| 509 |
-
"node": ">=18"
|
| 510 |
-
}
|
| 511 |
-
},
|
| 512 |
-
"node_modules/@esbuild/freebsd-x64": {
|
| 513 |
-
"version": "0.27.7",
|
| 514 |
-
"resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.27.7.tgz",
|
| 515 |
-
"integrity": "sha512-jOBDK5XEjA4m5IJK3bpAQF9/Lelu/Z9ZcdhTRLf4cajlB+8VEhFFRjWgfy3M1O4rO2GQ/b2dLwCUGpiF/eATNQ==",
|
| 516 |
-
"cpu": [
|
| 517 |
-
"x64"
|
| 518 |
-
],
|
| 519 |
-
"dev": true,
|
| 520 |
-
"license": "MIT",
|
| 521 |
-
"optional": true,
|
| 522 |
-
"os": [
|
| 523 |
-
"freebsd"
|
| 524 |
-
],
|
| 525 |
-
"engines": {
|
| 526 |
-
"node": ">=18"
|
| 527 |
-
}
|
| 528 |
-
},
|
| 529 |
-
"node_modules/@esbuild/linux-arm": {
|
| 530 |
-
"version": "0.27.7",
|
| 531 |
-
"resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.27.7.tgz",
|
| 532 |
-
"integrity": "sha512-RkT/YXYBTSULo3+af8Ib0ykH8u2MBh57o7q/DAs3lTJlyVQkgQvlrPTnjIzzRPQyavxtPtfg0EopvDyIt0j1rA==",
|
| 533 |
-
"cpu": [
|
| 534 |
-
"arm"
|
| 535 |
-
],
|
| 536 |
-
"dev": true,
|
| 537 |
-
"license": "MIT",
|
| 538 |
-
"optional": true,
|
| 539 |
-
"os": [
|
| 540 |
-
"linux"
|
| 541 |
-
],
|
| 542 |
-
"engines": {
|
| 543 |
-
"node": ">=18"
|
| 544 |
-
}
|
| 545 |
-
},
|
| 546 |
-
"node_modules/@esbuild/linux-arm64": {
|
| 547 |
-
"version": "0.27.7",
|
| 548 |
-
"resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.27.7.tgz",
|
| 549 |
-
"integrity": "sha512-RZPHBoxXuNnPQO9rvjh5jdkRmVizktkT7TCDkDmQ0W2SwHInKCAV95GRuvdSvA7w4VMwfCjUiPwDi0ZO6Nfe9A==",
|
| 550 |
-
"cpu": [
|
| 551 |
-
"arm64"
|
| 552 |
-
],
|
| 553 |
-
"dev": true,
|
| 554 |
-
"license": "MIT",
|
| 555 |
-
"optional": true,
|
| 556 |
-
"os": [
|
| 557 |
-
"linux"
|
| 558 |
-
],
|
| 559 |
-
"engines": {
|
| 560 |
-
"node": ">=18"
|
| 561 |
-
}
|
| 562 |
-
},
|
| 563 |
-
"node_modules/@esbuild/linux-ia32": {
|
| 564 |
-
"version": "0.27.7",
|
| 565 |
-
"resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.27.7.tgz",
|
| 566 |
-
"integrity": "sha512-GA48aKNkyQDbd3KtkplYWT102C5sn/EZTY4XROkxONgruHPU72l+gW+FfF8tf2cFjeHaRbWpOYa/uRBz/Xq1Pg==",
|
| 567 |
-
"cpu": [
|
| 568 |
-
"ia32"
|
| 569 |
-
],
|
| 570 |
-
"dev": true,
|
| 571 |
-
"license": "MIT",
|
| 572 |
-
"optional": true,
|
| 573 |
-
"os": [
|
| 574 |
-
"linux"
|
| 575 |
-
],
|
| 576 |
-
"engines": {
|
| 577 |
-
"node": ">=18"
|
| 578 |
-
}
|
| 579 |
-
},
|
| 580 |
-
"node_modules/@esbuild/linux-loong64": {
|
| 581 |
-
"version": "0.27.7",
|
| 582 |
-
"resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.27.7.tgz",
|
| 583 |
-
"integrity": "sha512-a4POruNM2oWsD4WKvBSEKGIiWQF8fZOAsycHOt6JBpZ+JN2n2JH9WAv56SOyu9X5IqAjqSIPTaJkqN8F7XOQ5Q==",
|
| 584 |
-
"cpu": [
|
| 585 |
-
"loong64"
|
| 586 |
-
],
|
| 587 |
-
"dev": true,
|
| 588 |
-
"license": "MIT",
|
| 589 |
-
"optional": true,
|
| 590 |
-
"os": [
|
| 591 |
-
"linux"
|
| 592 |
-
],
|
| 593 |
-
"engines": {
|
| 594 |
-
"node": ">=18"
|
| 595 |
-
}
|
| 596 |
-
},
|
| 597 |
-
"node_modules/@esbuild/linux-mips64el": {
|
| 598 |
-
"version": "0.27.7",
|
| 599 |
-
"resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.27.7.tgz",
|
| 600 |
-
"integrity": "sha512-KabT5I6StirGfIz0FMgl1I+R1H73Gp0ofL9A3nG3i/cYFJzKHhouBV5VWK1CSgKvVaG4q1RNpCTR2LuTVB3fIw==",
|
| 601 |
-
"cpu": [
|
| 602 |
-
"mips64el"
|
| 603 |
-
],
|
| 604 |
-
"dev": true,
|
| 605 |
-
"license": "MIT",
|
| 606 |
-
"optional": true,
|
| 607 |
-
"os": [
|
| 608 |
-
"linux"
|
| 609 |
-
],
|
| 610 |
-
"engines": {
|
| 611 |
-
"node": ">=18"
|
| 612 |
-
}
|
| 613 |
-
},
|
| 614 |
-
"node_modules/@esbuild/linux-ppc64": {
|
| 615 |
-
"version": "0.27.7",
|
| 616 |
-
"resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.27.7.tgz",
|
| 617 |
-
"integrity": "sha512-gRsL4x6wsGHGRqhtI+ifpN/vpOFTQtnbsupUF5R5YTAg+y/lKelYR1hXbnBdzDjGbMYjVJLJTd2OFmMewAgwlQ==",
|
| 618 |
-
"cpu": [
|
| 619 |
-
"ppc64"
|
| 620 |
-
],
|
| 621 |
-
"dev": true,
|
| 622 |
-
"license": "MIT",
|
| 623 |
-
"optional": true,
|
| 624 |
-
"os": [
|
| 625 |
-
"linux"
|
| 626 |
-
],
|
| 627 |
-
"engines": {
|
| 628 |
-
"node": ">=18"
|
| 629 |
-
}
|
| 630 |
-
},
|
| 631 |
-
"node_modules/@esbuild/linux-riscv64": {
|
| 632 |
-
"version": "0.27.7",
|
| 633 |
-
"resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.27.7.tgz",
|
| 634 |
-
"integrity": "sha512-hL25LbxO1QOngGzu2U5xeXtxXcW+/GvMN3ejANqXkxZ/opySAZMrc+9LY/WyjAan41unrR3YrmtTsUpwT66InQ==",
|
| 635 |
-
"cpu": [
|
| 636 |
-
"riscv64"
|
| 637 |
-
],
|
| 638 |
-
"dev": true,
|
| 639 |
-
"license": "MIT",
|
| 640 |
-
"optional": true,
|
| 641 |
-
"os": [
|
| 642 |
-
"linux"
|
| 643 |
-
],
|
| 644 |
-
"engines": {
|
| 645 |
-
"node": ">=18"
|
| 646 |
-
}
|
| 647 |
-
},
|
| 648 |
-
"node_modules/@esbuild/linux-s390x": {
|
| 649 |
-
"version": "0.27.7",
|
| 650 |
-
"resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.27.7.tgz",
|
| 651 |
-
"integrity": "sha512-2k8go8Ycu1Kb46vEelhu1vqEP+UeRVj2zY1pSuPdgvbd5ykAw82Lrro28vXUrRmzEsUV0NzCf54yARIK8r0fdw==",
|
| 652 |
-
"cpu": [
|
| 653 |
-
"s390x"
|
| 654 |
-
],
|
| 655 |
-
"dev": true,
|
| 656 |
-
"license": "MIT",
|
| 657 |
-
"optional": true,
|
| 658 |
-
"os": [
|
| 659 |
-
"linux"
|
| 660 |
-
],
|
| 661 |
-
"engines": {
|
| 662 |
-
"node": ">=18"
|
| 663 |
-
}
|
| 664 |
-
},
|
| 665 |
-
"node_modules/@esbuild/linux-x64": {
|
| 666 |
-
"version": "0.27.7",
|
| 667 |
-
"resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.27.7.tgz",
|
| 668 |
-
"integrity": "sha512-hzznmADPt+OmsYzw1EE33ccA+HPdIqiCRq7cQeL1Jlq2gb1+OyWBkMCrYGBJ+sxVzve2ZJEVeePbLM2iEIZSxA==",
|
| 669 |
-
"cpu": [
|
| 670 |
-
"x64"
|
| 671 |
-
],
|
| 672 |
-
"dev": true,
|
| 673 |
-
"license": "MIT",
|
| 674 |
-
"optional": true,
|
| 675 |
-
"os": [
|
| 676 |
-
"linux"
|
| 677 |
-
],
|
| 678 |
-
"engines": {
|
| 679 |
-
"node": ">=18"
|
| 680 |
-
}
|
| 681 |
-
},
|
| 682 |
-
"node_modules/@esbuild/netbsd-arm64": {
|
| 683 |
-
"version": "0.27.7",
|
| 684 |
-
"resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.27.7.tgz",
|
| 685 |
-
"integrity": "sha512-b6pqtrQdigZBwZxAn1UpazEisvwaIDvdbMbmrly7cDTMFnw/+3lVxxCTGOrkPVnsYIosJJXAsILG9XcQS+Yu6w==",
|
| 686 |
-
"cpu": [
|
| 687 |
-
"arm64"
|
| 688 |
-
],
|
| 689 |
-
"dev": true,
|
| 690 |
-
"license": "MIT",
|
| 691 |
-
"optional": true,
|
| 692 |
-
"os": [
|
| 693 |
-
"netbsd"
|
| 694 |
-
],
|
| 695 |
-
"engines": {
|
| 696 |
-
"node": ">=18"
|
| 697 |
-
}
|
| 698 |
-
},
|
| 699 |
-
"node_modules/@esbuild/netbsd-x64": {
|
| 700 |
-
"version": "0.27.7",
|
| 701 |
-
"resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.27.7.tgz",
|
| 702 |
-
"integrity": "sha512-OfatkLojr6U+WN5EDYuoQhtM+1xco+/6FSzJJnuWiUw5eVcicbyK3dq5EeV/QHT1uy6GoDhGbFpprUiHUYggrw==",
|
| 703 |
-
"cpu": [
|
| 704 |
-
"x64"
|
| 705 |
-
],
|
| 706 |
-
"dev": true,
|
| 707 |
-
"license": "MIT",
|
| 708 |
-
"optional": true,
|
| 709 |
-
"os": [
|
| 710 |
-
"netbsd"
|
| 711 |
-
],
|
| 712 |
-
"engines": {
|
| 713 |
-
"node": ">=18"
|
| 714 |
-
}
|
| 715 |
-
},
|
| 716 |
-
"node_modules/@esbuild/openbsd-arm64": {
|
| 717 |
-
"version": "0.27.7",
|
| 718 |
-
"resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.27.7.tgz",
|
| 719 |
-
"integrity": "sha512-AFuojMQTxAz75Fo8idVcqoQWEHIXFRbOc1TrVcFSgCZtQfSdc1RXgB3tjOn/krRHENUB4j00bfGjyl2mJrU37A==",
|
| 720 |
-
"cpu": [
|
| 721 |
-
"arm64"
|
| 722 |
-
],
|
| 723 |
-
"dev": true,
|
| 724 |
-
"license": "MIT",
|
| 725 |
-
"optional": true,
|
| 726 |
-
"os": [
|
| 727 |
-
"openbsd"
|
| 728 |
-
],
|
| 729 |
-
"engines": {
|
| 730 |
-
"node": ">=18"
|
| 731 |
-
}
|
| 732 |
-
},
|
| 733 |
-
"node_modules/@esbuild/openbsd-x64": {
|
| 734 |
-
"version": "0.27.7",
|
| 735 |
-
"resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.27.7.tgz",
|
| 736 |
-
"integrity": "sha512-+A1NJmfM8WNDv5CLVQYJ5PshuRm/4cI6WMZRg1by1GwPIQPCTs1GLEUHwiiQGT5zDdyLiRM/l1G0Pv54gvtKIg==",
|
| 737 |
-
"cpu": [
|
| 738 |
-
"x64"
|
| 739 |
-
],
|
| 740 |
-
"dev": true,
|
| 741 |
-
"license": "MIT",
|
| 742 |
-
"optional": true,
|
| 743 |
-
"os": [
|
| 744 |
-
"openbsd"
|
| 745 |
-
],
|
| 746 |
-
"engines": {
|
| 747 |
-
"node": ">=18"
|
| 748 |
-
}
|
| 749 |
-
},
|
| 750 |
-
"node_modules/@esbuild/openharmony-arm64": {
|
| 751 |
-
"version": "0.27.7",
|
| 752 |
-
"resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.27.7.tgz",
|
| 753 |
-
"integrity": "sha512-+KrvYb/C8zA9CU/g0sR6w2RBw7IGc5J2BPnc3dYc5VJxHCSF1yNMxTV5LQ7GuKteQXZtspjFbiuW5/dOj7H4Yw==",
|
| 754 |
-
"cpu": [
|
| 755 |
-
"arm64"
|
| 756 |
-
],
|
| 757 |
-
"dev": true,
|
| 758 |
-
"license": "MIT",
|
| 759 |
-
"optional": true,
|
| 760 |
-
"os": [
|
| 761 |
-
"openharmony"
|
| 762 |
-
],
|
| 763 |
-
"engines": {
|
| 764 |
-
"node": ">=18"
|
| 765 |
-
}
|
| 766 |
-
},
|
| 767 |
-
"node_modules/@esbuild/sunos-x64": {
|
| 768 |
-
"version": "0.27.7",
|
| 769 |
-
"resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.27.7.tgz",
|
| 770 |
-
"integrity": "sha512-ikktIhFBzQNt/QDyOL580ti9+5mL/YZeUPKU2ivGtGjdTYoqz6jObj6nOMfhASpS4GU4Q/Clh1QtxWAvcYKamA==",
|
| 771 |
-
"cpu": [
|
| 772 |
-
"x64"
|
| 773 |
-
],
|
| 774 |
-
"dev": true,
|
| 775 |
-
"license": "MIT",
|
| 776 |
-
"optional": true,
|
| 777 |
-
"os": [
|
| 778 |
-
"sunos"
|
| 779 |
-
],
|
| 780 |
-
"engines": {
|
| 781 |
-
"node": ">=18"
|
| 782 |
-
}
|
| 783 |
-
},
|
| 784 |
-
"node_modules/@esbuild/win32-arm64": {
|
| 785 |
-
"version": "0.27.7",
|
| 786 |
-
"resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.27.7.tgz",
|
| 787 |
-
"integrity": "sha512-7yRhbHvPqSpRUV7Q20VuDwbjW5kIMwTHpptuUzV+AA46kiPze5Z7qgt6CLCK3pWFrHeNfDd1VKgyP4O+ng17CA==",
|
| 788 |
-
"cpu": [
|
| 789 |
-
"arm64"
|
| 790 |
-
],
|
| 791 |
-
"dev": true,
|
| 792 |
-
"license": "MIT",
|
| 793 |
-
"optional": true,
|
| 794 |
-
"os": [
|
| 795 |
-
"win32"
|
| 796 |
-
],
|
| 797 |
-
"engines": {
|
| 798 |
-
"node": ">=18"
|
| 799 |
-
}
|
| 800 |
-
},
|
| 801 |
-
"node_modules/@esbuild/win32-ia32": {
|
| 802 |
-
"version": "0.27.7",
|
| 803 |
-
"resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.27.7.tgz",
|
| 804 |
-
"integrity": "sha512-SmwKXe6VHIyZYbBLJrhOoCJRB/Z1tckzmgTLfFYOfpMAx63BJEaL9ExI8x7v0oAO3Zh6D/Oi1gVxEYr5oUCFhw==",
|
| 805 |
-
"cpu": [
|
| 806 |
-
"ia32"
|
| 807 |
-
],
|
| 808 |
-
"dev": true,
|
| 809 |
-
"license": "MIT",
|
| 810 |
-
"optional": true,
|
| 811 |
-
"os": [
|
| 812 |
-
"win32"
|
| 813 |
-
],
|
| 814 |
-
"engines": {
|
| 815 |
-
"node": ">=18"
|
| 816 |
-
}
|
| 817 |
-
},
|
| 818 |
-
"node_modules/@esbuild/win32-x64": {
|
| 819 |
-
"version": "0.27.7",
|
| 820 |
-
"resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.27.7.tgz",
|
| 821 |
-
"integrity": "sha512-56hiAJPhwQ1R4i+21FVF7V8kSD5zZTdHcVuRFMW0hn753vVfQN8xlx4uOPT4xoGH0Z/oVATuR82AiqSTDIpaHg==",
|
| 822 |
-
"cpu": [
|
| 823 |
-
"x64"
|
| 824 |
-
],
|
| 825 |
-
"dev": true,
|
| 826 |
-
"license": "MIT",
|
| 827 |
-
"optional": true,
|
| 828 |
-
"os": [
|
| 829 |
-
"win32"
|
| 830 |
-
],
|
| 831 |
-
"engines": {
|
| 832 |
-
"node": ">=18"
|
| 833 |
-
}
|
| 834 |
-
},
|
| 835 |
"node_modules/@glideapps/glide-data-grid": {
|
| 836 |
"version": "6.0.3",
|
| 837 |
"resolved": "https://registry.npmjs.org/@glideapps/glide-data-grid/-/glide-data-grid-6.0.3.tgz",
|
|
@@ -2292,48 +1849,6 @@
|
|
| 2292 |
"integrity": "sha512-toUI84YS5YmxW219erniWD0CIVOo46xGKColeNQRgOzDorgBi1v4D71/OFzgD9GO2UGKIv1C3Sp8DAn0+j5w7A==",
|
| 2293 |
"license": "MIT"
|
| 2294 |
},
|
| 2295 |
-
"node_modules/esbuild": {
|
| 2296 |
-
"version": "0.27.7",
|
| 2297 |
-
"resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.27.7.tgz",
|
| 2298 |
-
"integrity": "sha512-IxpibTjyVnmrIQo5aqNpCgoACA/dTKLTlhMHihVHhdkxKyPO1uBBthumT0rdHmcsk9uMonIWS0m4FljWzILh3w==",
|
| 2299 |
-
"dev": true,
|
| 2300 |
-
"hasInstallScript": true,
|
| 2301 |
-
"license": "MIT",
|
| 2302 |
-
"bin": {
|
| 2303 |
-
"esbuild": "bin/esbuild"
|
| 2304 |
-
},
|
| 2305 |
-
"engines": {
|
| 2306 |
-
"node": ">=18"
|
| 2307 |
-
},
|
| 2308 |
-
"optionalDependencies": {
|
| 2309 |
-
"@esbuild/aix-ppc64": "0.27.7",
|
| 2310 |
-
"@esbuild/android-arm": "0.27.7",
|
| 2311 |
-
"@esbuild/android-arm64": "0.27.7",
|
| 2312 |
-
"@esbuild/android-x64": "0.27.7",
|
| 2313 |
-
"@esbuild/darwin-arm64": "0.27.7",
|
| 2314 |
-
"@esbuild/darwin-x64": "0.27.7",
|
| 2315 |
-
"@esbuild/freebsd-arm64": "0.27.7",
|
| 2316 |
-
"@esbuild/freebsd-x64": "0.27.7",
|
| 2317 |
-
"@esbuild/linux-arm": "0.27.7",
|
| 2318 |
-
"@esbuild/linux-arm64": "0.27.7",
|
| 2319 |
-
"@esbuild/linux-ia32": "0.27.7",
|
| 2320 |
-
"@esbuild/linux-loong64": "0.27.7",
|
| 2321 |
-
"@esbuild/linux-mips64el": "0.27.7",
|
| 2322 |
-
"@esbuild/linux-ppc64": "0.27.7",
|
| 2323 |
-
"@esbuild/linux-riscv64": "0.27.7",
|
| 2324 |
-
"@esbuild/linux-s390x": "0.27.7",
|
| 2325 |
-
"@esbuild/linux-x64": "0.27.7",
|
| 2326 |
-
"@esbuild/netbsd-arm64": "0.27.7",
|
| 2327 |
-
"@esbuild/netbsd-x64": "0.27.7",
|
| 2328 |
-
"@esbuild/openbsd-arm64": "0.27.7",
|
| 2329 |
-
"@esbuild/openbsd-x64": "0.27.7",
|
| 2330 |
-
"@esbuild/openharmony-arm64": "0.27.7",
|
| 2331 |
-
"@esbuild/sunos-x64": "0.27.7",
|
| 2332 |
-
"@esbuild/win32-arm64": "0.27.7",
|
| 2333 |
-
"@esbuild/win32-ia32": "0.27.7",
|
| 2334 |
-
"@esbuild/win32-x64": "0.27.7"
|
| 2335 |
-
}
|
| 2336 |
-
},
|
| 2337 |
"node_modules/escalade": {
|
| 2338 |
"version": "3.2.0",
|
| 2339 |
"resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz",
|
|
|
|
| 25 |
"@types/react": "^18.3.12",
|
| 26 |
"@types/react-dom": "^18.3.1",
|
| 27 |
"@vitejs/plugin-react": "^6.0.3",
|
|
|
|
| 28 |
"openapi-typescript": "^7.13.0",
|
| 29 |
"oxlint": "^1.71.0",
|
| 30 |
"typescript": "~6.0.2",
|
|
|
|
| 389 |
"integrity": "sha512-30FAj7/EoJ5mwVPOWhAyCX+FPfMDrVecJAM+Iw9NRoSl4BBAQeqj4cApHHUXOVvIPgLVDsCFoz/hGD+5QQD1GQ==",
|
| 390 |
"license": "MIT"
|
| 391 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 392 |
"node_modules/@glideapps/glide-data-grid": {
|
| 393 |
"version": "6.0.3",
|
| 394 |
"resolved": "https://registry.npmjs.org/@glideapps/glide-data-grid/-/glide-data-grid-6.0.3.tgz",
|
|
|
|
| 1849 |
"integrity": "sha512-toUI84YS5YmxW219erniWD0CIVOo46xGKColeNQRgOzDorgBi1v4D71/OFzgD9GO2UGKIv1C3Sp8DAn0+j5w7A==",
|
| 1850 |
"license": "MIT"
|
| 1851 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1852 |
"node_modules/escalade": {
|
| 1853 |
"version": "3.2.0",
|
| 1854 |
"resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz",
|
web/package-lock.linux.sha256
CHANGED
|
@@ -1,2 +1,2 @@
|
|
| 1 |
-
|
| 2 |
-
# resolved on linux/x64
|
|
|
|
| 1 |
+
31cb494538043db42bdf698846c2262f5eb234fb686aa03d9cf3ab1cbfdbe2b8 package-lock.json
|
| 2 |
+
# resolved on linux/x64 in the SDLC 72 lane A HF container on 2026-10-09 (S72-T05, npm install --save-exact @phosphor-icons/react@2.1.10); npm ci --ignore-scripts passed there (228 packages). Regenerate ONLY with lockfile_linux.py (W-170).
|
web/package.json
CHANGED
|
@@ -39,7 +39,6 @@
|
|
| 39 |
"@types/react": "^18.3.12",
|
| 40 |
"@types/react-dom": "^18.3.1",
|
| 41 |
"@vitejs/plugin-react": "^6.0.3",
|
| 42 |
-
"esbuild": "^0.27.0",
|
| 43 |
"openapi-typescript": "^7.13.0",
|
| 44 |
"oxlint": "^1.71.0",
|
| 45 |
"typescript": "~6.0.2",
|
|
|
|
| 39 |
"@types/react": "^18.3.12",
|
| 40 |
"@types/react-dom": "^18.3.1",
|
| 41 |
"@vitejs/plugin-react": "^6.0.3",
|
|
|
|
| 42 |
"openapi-typescript": "^7.13.0",
|
| 43 |
"oxlint": "^1.71.0",
|
| 44 |
"typescript": "~6.0.2",
|
web/public/AIOS_DEPLOY_RECEIPT
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
|
|
|
|
| 1 |
+
f4813286cc57af9ab94a458dafeabfd9f77075e08fa73b573fd34b3ec376779d
|
web/src/apps/amazon-profit/AppView.tsx
CHANGED
|
@@ -2,7 +2,7 @@ import { useCallback, useEffect, useMemo, useState } from "react";
|
|
| 2 |
import { API_V1, CREDENTIALS, ROWS_STALE_EVENT, signal } from "../../apiContract";
|
| 3 |
import { ACCESS_REFUSED_MESSAGE, AppTableGrid, clearTopicRowsCache, dropTableRowsCache,
|
| 4 |
OverlayProvider, LP_BLUE_DEEP, LP_GREEN_DEEP } from "../../customer-grid/seam";
|
| 5 |
-
import type {
|
| 6 |
import { AMAZON_DATA_ENTRY_TABLE, AMAZON_PROFIT_TABLE, AmazonToolbar, COMMAND_CENTER_TABLES,
|
| 7 |
CommandCenterTabStrip, CommandCenterTableContext } from "./AmazonPeriodPicker";
|
| 8 |
import { LogoLoader } from "../../ui/LogoLoader";
|
|
@@ -11,36 +11,12 @@ import PicturePopup from "./PicturePopup";
|
|
| 11 |
import { ProjectsPopup, ProjectsPage, projectsTargetFromRow } from "./Projects";
|
| 12 |
import type { ProjectsTarget } from "./Projects";
|
| 13 |
import { CompetitorsPopup, competitorsTargetFromRow } from "./Competitors";
|
| 14 |
-
import GridFilter, { SIGNAL_SORTS } from "./pdp/GridFilter";
|
| 15 |
-
import HistoryPopup from "./pdp/HistoryPopup";
|
| 16 |
-
import PdpUpdatePopup, { type PdpListing, type PdpNavigation } from "./pdp/PdpUpdatePopup";
|
| 17 |
-
import WorkflowWindow from "./pdp/WorkflowWindow";
|
| 18 |
-
import type { PdpProjectNavigation } from "./pdp/ContinueWorkflow";
|
| 19 |
-
import type { PdpProjectPane } from "./pdp/contract";
|
| 20 |
import "../../styles/sdlc81/command-center-app.css";
|
| 21 |
-
import "../../styles/sdlc86/project-manager-pdp.css";
|
| 22 |
|
| 23 |
const PROJECT_PROGRESS_COLORS = { inProgress: LP_BLUE_DEEP, completed: LP_GREEN_DEEP };
|
| 24 |
const PROJECT_PROGRESS_TABLES = new Set([AMAZON_PROFIT_TABLE, "ut_amazon_asin_profit", "ut_amazon_parent_profit"]);
|
| 25 |
/** The App's 56 px rows (R5). Table keeps its own default. */
|
| 26 |
const COMMAND_CENTER_COLUMNS = { rowHeight: 56 };
|
| 27 |
-
const ASIN_COLUMNS = { rowHeight: 56, headerTone: { projects: "primary", pdp_update: "primary", history: "primary" },
|
| 28 |
-
widths: { asin: 150, projects: 210, pdp_update: 170, trend_score: 150, search_trend: 210,
|
| 29 |
-
market_share_trend: 240, click_share_trend: 240, exposure: 190, funnel_score: 150, history: 150 } } as const;
|
| 30 |
-
const noReport = (reasonKey: string) => ({ reasonKey, equals: "No search report", text: "No search report" });
|
| 31 |
-
const signalTrend = (key: string, share: boolean): AppCellDraw => ({ kind: "trend", seriesKey: key + "_series",
|
| 32 |
-
valueKey: key + "_now", changeKey: key, format: share ? "percent" : "number", showValue: share,
|
| 33 |
-
omitMissingValue: true, valuePrecision: "adaptive", changeDecimals: 0, changeNeutralBand: .5, fitLabels: true, barTone: "primary",
|
| 34 |
-
unavailable: noReport(key + "_reason") });
|
| 35 |
-
const SIGNAL_CELLS: AppTableHost["cellRenderers"] = {
|
| 36 |
-
trend_score: { draw: { kind: "ring", labelPosition: "after", unavailable: noReport("trend_score_reason") } },
|
| 37 |
-
search_trend: { draw: signalTrend("search_trend", false) },
|
| 38 |
-
market_share_trend: { draw: signalTrend("market_share_trend", true) },
|
| 39 |
-
click_share_trend: { draw: signalTrend("click_share_trend", true) },
|
| 40 |
-
exposure: { draw: { kind: "badge", subKey: "exposure_label", emptyText: "None", unavailable: noReport("exposure_reason") } },
|
| 41 |
-
funnel_score: { draw: { kind: "score", neutralRange: [4.5, 5.5], outerRange: [3, 7], emptyText: "Few",
|
| 42 |
-
unavailable: noReport("funnel_score_reason") } },
|
| 43 |
-
};
|
| 44 |
|
| 45 |
/** Admission belongs to the selected tab's fresh probe. */
|
| 46 |
type TableProbe = { table: string; status: "ready"; tables: Record<string, boolean> }
|
|
@@ -48,17 +24,12 @@ type TableProbe = { table: string; status: "ready"; tables: Record<string, boole
|
|
| 48 |
|
| 49 |
// The Command Center App body: the Table module's own Grid, hosted under the App frame.
|
| 50 |
// The tab strip switches `table` through the context, so the URL stays on the App.
|
| 51 |
-
export default function ProjectManagementView(
|
| 52 |
const [table, setTable] = useState(AMAZON_PROFIT_TABLE);
|
| 53 |
const [probe, setProbe] = useState<TableProbe>(null);
|
| 54 |
const [picture, setPicture] = useState<{ ref: string; label: string } | null>(null);
|
| 55 |
-
const [projects, setProjects] = useState<
|
| 56 |
-
selectedProjectId?: string; originListing?: PdpListing; initialPane?: PdpProjectPane;
|
| 57 |
-
}) | null>(null);
|
| 58 |
const [competitors, setCompetitors] = useState<ProjectsTarget | null>(null);
|
| 59 |
-
const [history, setHistory] = useState<Row | null>(null);
|
| 60 |
-
const [pdp, setPdp] = useState<PdpListing | null>(null);
|
| 61 |
-
const [workflow, setWorkflow] = useState<{ id: string; listing: PdpListing; resumeRevision?: number } | null>(null);
|
| 62 |
const [gridRevision, setGridRevision] = useState(0);
|
| 63 |
const [entryEmpty, setEntryEmpty] = useState<{ table: string; empty: boolean } | null>(null);
|
| 64 |
const openTable = useCallback((next: string) => {
|
|
@@ -69,9 +40,6 @@ export default function ProjectManagementView(props: AppViewProps & PdpNavigatio
|
|
| 69 |
setPicture(null);
|
| 70 |
setProjects(null);
|
| 71 |
setCompetitors(null);
|
| 72 |
-
setHistory(null);
|
| 73 |
-
setPdp(null);
|
| 74 |
-
setWorkflow(null);
|
| 75 |
setTable(next);
|
| 76 |
}, [table]);
|
| 77 |
const reloadGrid = useCallback(() => {
|
|
@@ -120,37 +88,12 @@ export default function ProjectManagementView(props: AppViewProps & PdpNavigatio
|
|
| 120 |
}, [dataEntry, admitted, table]);
|
| 121 |
const entryChecked = !dataEntry || notBuilt || entryEmpty?.table === table;
|
| 122 |
const allComplete = dataEntry && admitted && entryEmpty?.table === table && entryEmpty.empty;
|
| 123 |
-
const openProject = (id: string, asin: string, listing: PdpListing, initialPane?: PdpProjectPane) => {
|
| 124 |
-
if (props.onOpenProject) { props.onOpenProject(id, asin); return; }
|
| 125 |
-
setPdp(null);
|
| 126 |
-
setProjects({ grain: "asin", rowKey: asin, title: listing.title, selectedProjectId: id, originListing: listing, initialPane });
|
| 127 |
-
};
|
| 128 |
-
const resumeWorkflow = () => {
|
| 129 |
-
setProjects(null);
|
| 130 |
-
setWorkflow(current => current ? { ...current, resumeRevision: (current.resumeRevision ?? 0) + 1 } : current);
|
| 131 |
-
};
|
| 132 |
-
const projectNavigation: PdpProjectNavigation = { onOpenWorkflow: (next, item, target) => {
|
| 133 |
-
if (workflow?.id === next.id && workflow.listing.asin === item.asin) {
|
| 134 |
-
resumeWorkflow();
|
| 135 |
-
return;
|
| 136 |
-
}
|
| 137 |
-
setProjects(null);
|
| 138 |
-
if (props.onOpenWorkflow) { props.onOpenWorkflow(next.id, item.asin); return; }
|
| 139 |
-
setWorkflow({ id: next.id, listing: { asin: item.asin, title: target?.title ?? "", sku: target?.sku ?? null, brand: null } });
|
| 140 |
-
} };
|
| 141 |
// C2: progress colours, button doors and the Picture popup, declared once per table.
|
| 142 |
// Only the Projects table keeps New record (`append`).
|
| 143 |
const cellRenderers = useMemo<AppTableHost["cellRenderers"]>(() => ({
|
| 144 |
-
...(table === "ut_amazon_asin_profit" ? SIGNAL_CELLS : {}),
|
| 145 |
"*": { progressColors: PROJECT_PROGRESS_TABLES.has(table) ? PROJECT_PROGRESS_COLORS : undefined },
|
| 146 |
projects: { button: true, activate: row => setProjects(projectsTargetFromRow(table, row)) },
|
| 147 |
competitors: { button: true, activate: row => setCompetitors(competitorsTargetFromRow(table, row)) },
|
| 148 |
-
history: { button: true, activate: row => setHistory(row) },
|
| 149 |
-
pdp_update: { button: true, activate: row => {
|
| 150 |
-
if (!row.asin) return false;
|
| 151 |
-
setPdp({ asin: String(row.asin), title: String(row.title || ""),
|
| 152 |
-
sku: row.sku ? String(row.sku) : null, brand: row.brand ? String(row.brand) : null });
|
| 153 |
-
} },
|
| 154 |
"type:image": { activate: (record, field) => {
|
| 155 |
const ref = String(record[field.key] ?? "").trim();
|
| 156 |
if (!ref) return false;
|
|
@@ -172,35 +115,17 @@ export default function ProjectManagementView(props: AppViewProps & PdpNavigatio
|
|
| 172 |
</div>
|
| 173 |
: table === "ut_amazon_projects" ? <div style={{ display: "flex", flexDirection: "column", flex: "1 1 0", minHeight: 0, padding: "4px 12px 0" }}>
|
| 174 |
<CommandCenterTabStrip database={table} onOpenTable={openTable} />
|
| 175 |
-
<ProjectsPage
|
| 176 |
</div>
|
| 177 |
: !entryChecked ? <LogoLoader data-app-reason="loading" /> :
|
| 178 |
<AppTableGrid key={`${table}:${gridRevision}`}
|
| 179 |
data={{ database: table, append: COMMAND_CENTER_TABLES[table] === "projects" }}
|
| 180 |
-
columns={
|
| 181 |
-
history={false} filterPanel={table === "ut_amazon_asin_profit" ? props => <GridFilter {...props} /> : undefined}
|
| 182 |
-
sortChoices={table === "ut_amazon_asin_profit" ? SIGNAL_SORTS : undefined}
|
| 183 |
toolbar={<AmazonToolbar database={table} />} />}
|
| 184 |
{picture && <PicturePopup pictureRef={picture.ref} label={picture.label}
|
| 185 |
onClose={() => setPicture(null)} />}
|
| 186 |
-
{projects && <ProjectsPopup {...projects}
|
| 187 |
-
onClose={projects.initialPane ? resumeWorkflow : () => setProjects(null)} onChanged={reloadGrid}
|
| 188 |
-
backLabel={projects.initialPane ? "Back to workflow" : projects.originListing ? "Back to projects" : undefined}
|
| 189 |
-
onBack={projects.initialPane ? resumeWorkflow : projects.originListing
|
| 190 |
-
? () => { setPdp(projects.originListing ?? null); setProjects(null); setWorkflow(null); } : undefined} />}
|
| 191 |
{competitors && <CompetitorsPopup {...competitors} onClose={() => setCompetitors(null)} onChanged={reloadGrid} />}
|
| 192 |
-
{history && <HistoryPopup table={table} row={history} onClose={() => setHistory(null)} onChanged={reloadGrid} />}
|
| 193 |
-
{pdp && <PdpUpdatePopup listing={pdp} onClose={() => setPdp(null)} onChanged={reloadGrid}
|
| 194 |
-
onOpenWorkflow={props.onOpenWorkflow ?? ((id, asin) => {
|
| 195 |
-
setWorkflow({ id, listing: { ...pdp, asin } }); setPdp(null);
|
| 196 |
-
})} onAddWorkflow={props.onAddWorkflow}
|
| 197 |
-
onOpenProject={(id, asin) => openProject(id, asin, pdp)} />}
|
| 198 |
-
{workflow && <WorkflowWindow key={`${workflow.id}:${workflow.listing.asin}`} workflowId={workflow.id} listing={workflow.listing}
|
| 199 |
-
resumeRevision={workflow.resumeRevision}
|
| 200 |
-
onClose={() => setWorkflow(null)} onChanged={reloadGrid}
|
| 201 |
-
onBack={() => { setPdp(workflow.listing); setWorkflow(null); }}
|
| 202 |
-
onOpenProject={() => openProject(workflow.id, workflow.listing.asin, workflow.listing)}
|
| 203 |
-
onOpenProjectPane={pane => openProject(workflow.id, workflow.listing.asin, workflow.listing, pane)} />}
|
| 204 |
</OverlayProvider>
|
| 205 |
</CommandCenterTableContext.Provider>;
|
| 206 |
}
|
|
|
|
| 2 |
import { API_V1, CREDENTIALS, ROWS_STALE_EVENT, signal } from "../../apiContract";
|
| 3 |
import { ACCESS_REFUSED_MESSAGE, AppTableGrid, clearTopicRowsCache, dropTableRowsCache,
|
| 4 |
OverlayProvider, LP_BLUE_DEEP, LP_GREEN_DEEP } from "../../customer-grid/seam";
|
| 5 |
+
import type { AppTableHost, AppViewProps } from "../../customer-grid/seam";
|
| 6 |
import { AMAZON_DATA_ENTRY_TABLE, AMAZON_PROFIT_TABLE, AmazonToolbar, COMMAND_CENTER_TABLES,
|
| 7 |
CommandCenterTabStrip, CommandCenterTableContext } from "./AmazonPeriodPicker";
|
| 8 |
import { LogoLoader } from "../../ui/LogoLoader";
|
|
|
|
| 11 |
import { ProjectsPopup, ProjectsPage, projectsTargetFromRow } from "./Projects";
|
| 12 |
import type { ProjectsTarget } from "./Projects";
|
| 13 |
import { CompetitorsPopup, competitorsTargetFromRow } from "./Competitors";
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 14 |
import "../../styles/sdlc81/command-center-app.css";
|
|
|
|
| 15 |
|
| 16 |
const PROJECT_PROGRESS_COLORS = { inProgress: LP_BLUE_DEEP, completed: LP_GREEN_DEEP };
|
| 17 |
const PROJECT_PROGRESS_TABLES = new Set([AMAZON_PROFIT_TABLE, "ut_amazon_asin_profit", "ut_amazon_parent_profit"]);
|
| 18 |
/** The App's 56 px rows (R5). Table keeps its own default. */
|
| 19 |
const COMMAND_CENTER_COLUMNS = { rowHeight: 56 };
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 20 |
|
| 21 |
/** Admission belongs to the selected tab's fresh probe. */
|
| 22 |
type TableProbe = { table: string; status: "ready"; tables: Record<string, boolean> }
|
|
|
|
| 24 |
|
| 25 |
// The Command Center App body: the Table module's own Grid, hosted under the App frame.
|
| 26 |
// The tab strip switches `table` through the context, so the URL stays on the App.
|
| 27 |
+
export default function ProjectManagementView(_props: AppViewProps) {
|
| 28 |
const [table, setTable] = useState(AMAZON_PROFIT_TABLE);
|
| 29 |
const [probe, setProbe] = useState<TableProbe>(null);
|
| 30 |
const [picture, setPicture] = useState<{ ref: string; label: string } | null>(null);
|
| 31 |
+
const [projects, setProjects] = useState<ProjectsTarget | null>(null);
|
|
|
|
|
|
|
| 32 |
const [competitors, setCompetitors] = useState<ProjectsTarget | null>(null);
|
|
|
|
|
|
|
|
|
|
| 33 |
const [gridRevision, setGridRevision] = useState(0);
|
| 34 |
const [entryEmpty, setEntryEmpty] = useState<{ table: string; empty: boolean } | null>(null);
|
| 35 |
const openTable = useCallback((next: string) => {
|
|
|
|
| 40 |
setPicture(null);
|
| 41 |
setProjects(null);
|
| 42 |
setCompetitors(null);
|
|
|
|
|
|
|
|
|
|
| 43 |
setTable(next);
|
| 44 |
}, [table]);
|
| 45 |
const reloadGrid = useCallback(() => {
|
|
|
|
| 88 |
}, [dataEntry, admitted, table]);
|
| 89 |
const entryChecked = !dataEntry || notBuilt || entryEmpty?.table === table;
|
| 90 |
const allComplete = dataEntry && admitted && entryEmpty?.table === table && entryEmpty.empty;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 91 |
// C2: progress colours, button doors and the Picture popup, declared once per table.
|
| 92 |
// Only the Projects table keeps New record (`append`).
|
| 93 |
const cellRenderers = useMemo<AppTableHost["cellRenderers"]>(() => ({
|
|
|
|
| 94 |
"*": { progressColors: PROJECT_PROGRESS_TABLES.has(table) ? PROJECT_PROGRESS_COLORS : undefined },
|
| 95 |
projects: { button: true, activate: row => setProjects(projectsTargetFromRow(table, row)) },
|
| 96 |
competitors: { button: true, activate: row => setCompetitors(competitorsTargetFromRow(table, row)) },
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 97 |
"type:image": { activate: (record, field) => {
|
| 98 |
const ref = String(record[field.key] ?? "").trim();
|
| 99 |
if (!ref) return false;
|
|
|
|
| 115 |
</div>
|
| 116 |
: table === "ut_amazon_projects" ? <div style={{ display: "flex", flexDirection: "column", flex: "1 1 0", minHeight: 0, padding: "4px 12px 0" }}>
|
| 117 |
<CommandCenterTabStrip database={table} onOpenTable={openTable} />
|
| 118 |
+
<ProjectsPage />
|
| 119 |
</div>
|
| 120 |
: !entryChecked ? <LogoLoader data-app-reason="loading" /> :
|
| 121 |
<AppTableGrid key={`${table}:${gridRevision}`}
|
| 122 |
data={{ database: table, append: COMMAND_CENTER_TABLES[table] === "projects" }}
|
| 123 |
+
columns={COMMAND_CENTER_COLUMNS} cellRenderers={cellRenderers}
|
|
|
|
|
|
|
| 124 |
toolbar={<AmazonToolbar database={table} />} />}
|
| 125 |
{picture && <PicturePopup pictureRef={picture.ref} label={picture.label}
|
| 126 |
onClose={() => setPicture(null)} />}
|
| 127 |
+
{projects && <ProjectsPopup {...projects} onClose={() => setProjects(null)} onChanged={reloadGrid} />}
|
|
|
|
|
|
|
|
|
|
|
|
|
| 128 |
{competitors && <CompetitorsPopup {...competitors} onClose={() => setCompetitors(null)} onChanged={reloadGrid} />}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 129 |
</OverlayProvider>
|
| 130 |
</CommandCenterTableContext.Provider>;
|
| 131 |
}
|
web/src/apps/amazon-profit/ProjectOverview.tsx
CHANGED
|
@@ -11,8 +11,6 @@ import { X } from "@phosphor-icons/react/dist/csr/X";
|
|
| 11 |
import { useEscapeClose } from "../../ui/useEscapeClose";
|
| 12 |
import ProjectTasks from "./ProjectTasks";
|
| 13 |
import ProjectSkuPicker from "./ProjectSkuPicker";
|
| 14 |
-
import { PdpApprovalRow, PdpOverviewRows, PdpStep, type ProjectPdp } from "./pdp/ContinueWorkflow";
|
| 15 |
-
import { removeItems } from "./pdp/api";
|
| 16 |
import {
|
| 17 |
EMPTY_FILTER, GRAINS, ProductToolbar, SelectBox, TABLES, normalizeProducts, number, readPage, scopedNodes,
|
| 18 |
type Field, type FilterTree, type Row, type Sorts,
|
|
@@ -238,12 +236,11 @@ function ProjectDeleteDialog({ name, busy, error, onCancel, onConfirm }: {
|
|
| 238 |
}
|
| 239 |
|
| 240 |
export default function ProjectOverview({ id, detail, canEdit: mayEdit, onChanged, onDeleted, onCancel,
|
| 241 |
-
error, setError, setDetail, setTick, formId, active, rowScope = null, onClearRowScope
|
| 242 |
id: string; detail: Detail; canEdit: boolean; onChanged: () => void; onDeleted: () => void; onCancel: () => void;
|
| 243 |
error: string; setError: Dispatch<SetStateAction<string>>; setDetail: Dispatch<SetStateAction<Detail | null>>;
|
| 244 |
setTick: Dispatch<SetStateAction<number>>; formId: string; active: boolean;
|
| 245 |
rowScope?: RowScope | null; onClearRowScope?: () => void;
|
| 246 |
-
pdp?: ProjectPdp;
|
| 247 |
}) {
|
| 248 |
const [draft, setDraft] = useState<OverviewDraft>(() => overviewDraftOf(detail.project));
|
| 249 |
const [people, setPeople] = useState<ProjectPerson[]>([]);
|
|
@@ -321,12 +318,6 @@ export default function ProjectOverview({ id, detail, canEdit: mayEdit, onChange
|
|
| 321 |
setBusy(true);
|
| 322 |
const fallback = "Products were not removed.";
|
| 323 |
try {
|
| 324 |
-
if (pdp) {
|
| 325 |
-
if (!pdp.detail?.canEdit) throw new Error("The workflow is unavailable. Reload the project.");
|
| 326 |
-
pdp.accept((await removeItems(id, pairs.map(pair => pair.key))).workflow);
|
| 327 |
-
setTick(value => value + 1);
|
| 328 |
-
return { refused: [] };
|
| 329 |
-
}
|
| 330 |
const answer = await call<RemoveAnswer>(`/${enc(id)}/targets/remove`,
|
| 331 |
{ method: "POST", body: { targets: pairs } }, fallback);
|
| 332 |
if (!answer?.project || !Array.isArray(answer.targets)) throw new Error("The remove answer is incomplete. Reload the project.");
|
|
@@ -352,10 +343,10 @@ export default function ProjectOverview({ id, detail, canEdit: mayEdit, onChange
|
|
| 352 |
writing.current = false;
|
| 353 |
setBusy(false);
|
| 354 |
}
|
| 355 |
-
}, [detail?.canEdit, id, mayEdit, onChanged, setDetail, setError, setTick
|
| 356 |
|
| 357 |
const { project, targets } = detail;
|
| 358 |
-
const canEdit = mayEdit && detail.canEdit
|
| 359 |
const deleteOpen = confirming && canEdit && active;
|
| 360 |
useEffect(() => {
|
| 361 |
if (!canEdit || !active) { setConfirming(false); setAdding(false); }
|
|
@@ -469,7 +460,6 @@ export default function ProjectOverview({ id, detail, canEdit: mayEdit, onChange
|
|
| 469 |
</section>
|
| 470 |
|
| 471 |
<div className="lp-proj-setup-fields">
|
| 472 |
-
{project.type === "pdp" && pdp ? <PdpOverviewRows pdp={pdp} /> : null}
|
| 473 |
<SetupRow field="status" label="Status" error={fieldErrors.status} readOnly={!canEdit}>
|
| 474 |
{canEdit
|
| 475 |
? <StatusPicker value={draft.status} disabled={busy} onChange={value => set("status", value)} />
|
|
@@ -492,7 +482,6 @@ export default function ProjectOverview({ id, detail, canEdit: mayEdit, onChange
|
|
| 492 |
onChange={values => { set("ownerUsername", values[0] ?? null); set("owner", null); }} /> : project.owner || project.ownerUsername
|
| 493 |
? <Owner name={project.owner || project.ownerUsername || null} /> : <span className="lp-proj-muted">No owner</span>}
|
| 494 |
</SetupRow>
|
| 495 |
-
{project.type === "pdp" && pdp ? <PdpApprovalRow pdp={pdp} canEdit={canEdit} /> : null}
|
| 496 |
<SetupRow field="contributors" label="Contributors" error={fieldErrors.contributors} readOnly={!canEdit}>
|
| 497 |
<div className="lp-proj-contributors">
|
| 498 |
{contributors.map(person => <span key={person.username} className="lp-proj-contributor">
|
|
@@ -566,7 +555,7 @@ export default function ProjectOverview({ id, detail, canEdit: mayEdit, onChange
|
|
| 566 |
: null}
|
| 567 |
</div>
|
| 568 |
{canEdit && active && adding
|
| 569 |
-
? <ProjectSkuPicker projectId={id} existing={targets} canEdit={canEdit && !busy} onClose={() => setAdding(false)}
|
| 570 |
onAdded={answer => {
|
| 571 |
setDetail(current => current ? { ...current, project: answer.project, targets: answer.targets,
|
| 572 |
cause: answer.cause } : current);
|
|
@@ -577,7 +566,7 @@ export default function ProjectOverview({ id, detail, canEdit: mayEdit, onChange
|
|
| 577 |
: null}
|
| 578 |
{targets.length
|
| 579 |
? <ProjectSkuList targets={targets} scope={scope} canEdit={canEdit} busy={busy} active={active}
|
| 580 |
-
onRemove={removeTargets}
|
| 581 |
: null}
|
| 582 |
</section>
|
| 583 |
</SetupRow>
|
|
@@ -665,10 +654,9 @@ async function readLinked(table: string, grain: ProjectGrain, tree: FilterTree,
|
|
| 665 |
}
|
| 666 |
}
|
| 667 |
|
| 668 |
-
function ProjectSkuList({ targets: linked, scope, canEdit, busy, active, onRemove
|
| 669 |
targets: Target[]; scope: (RowScope & { clear: () => void }) | null; canEdit: boolean; busy: boolean; active: boolean;
|
| 670 |
onRemove: (pairs: TargetPair[]) => Promise<RemoveOutcome | null>;
|
| 671 |
-
pdp?: ProjectPdp;
|
| 672 |
}) {
|
| 673 |
const id = useId();
|
| 674 |
const [chosen, setChosen] = useState<ProjectGrain>("sku");
|
|
@@ -698,14 +686,13 @@ function ProjectSkuList({ targets: linked, scope, canEdit, busy, active, onRemov
|
|
| 698 |
</div> : null}
|
| 699 |
<SkuGrainList key={grain} grain={grain} panelId={`${id}-panel`} tabId={tabbed ? `${id}-tab-${grain}` : null}
|
| 700 |
targets={targets.filter(target => target.grain === grain)} canEdit={canEdit} busy={busy} active={active}
|
| 701 |
-
onRemove={onRemove}
|
| 702 |
</div>;
|
| 703 |
}
|
| 704 |
|
| 705 |
-
function SkuGrainList({ grain, panelId, tabId, targets, canEdit, busy, active, onRemove
|
| 706 |
grain: ProjectGrain; panelId: string; tabId: string | null; targets: Target[]; canEdit: boolean;
|
| 707 |
busy: boolean; active: boolean; onRemove: (pairs: TargetPair[]) => Promise<RemoveOutcome | null>;
|
| 708 |
-
pdp?: ProjectPdp;
|
| 709 |
}) {
|
| 710 |
const table = TABLES[grain];
|
| 711 |
const label = GRAIN_LABEL[grain];
|
|
@@ -799,7 +786,7 @@ function SkuGrainList({ grain, panelId, tabId, targets, canEdit, busy, active, o
|
|
| 799 |
: new Set(Array.from(current).filter(key => shown.has(key))));
|
| 800 |
}, [term, blocked, shownSignature]);
|
| 801 |
const columns = LIST_COLUMNS[grain];
|
| 802 |
-
const span = (canEdit ? 1 : 0) + 1 + columns.length
|
| 803 |
const resetView = () => { setSelected(new Set()); setRefused([]); setRemoveError(""); };
|
| 804 |
const clearView = () => { resetView(); setFilters(EMPTY_FILTER); setSorts([]); setGroup(null); setSearch(""); };
|
| 805 |
const viewSet = filtered || sorts.length > 0 || group !== null || selected.size > 0 || search !== "";
|
|
@@ -864,7 +851,6 @@ function SkuGrainList({ grain, panelId, tabId, targets, canEdit, busy, active, o
|
|
| 864 |
{canEdit ? <th scope="col" className="lp-proj-skus-check">{boxFor(visibleKeys, `Select all shown ${plural}`)}</th> : null}
|
| 865 |
<th scope="col">Product</th>
|
| 866 |
{columns.map(column => <th scope="col" key={column.key}>{column.label}</th>)}
|
| 867 |
-
{pdp ? <><th scope="col">Step</th><th scope="col"><span className="lp-proj-sku-sr">Workflow</span></th></> : null}
|
| 868 |
</tr></thead>
|
| 869 |
<tbody>{groups.map(([value, members]) => <Fragment key={value}>
|
| 870 |
{group !== null ? <tr className="lp-proj-sku-group"><th scope="rowgroup" colSpan={span}>
|
|
@@ -887,7 +873,6 @@ function SkuGrainList({ grain, panelId, tabId, targets, canEdit, busy, active, o
|
|
| 887 |
return <td key={column.key} className="lp-proj-sku-code" title={shown || undefined}>
|
| 888 |
{shown || <span className="lp-proj-muted">Not set</span>}</td>;
|
| 889 |
})}
|
| 890 |
-
{pdp ? <PdpSkuCells pdp={pdp} target={row.target} /> : null}
|
| 891 |
</tr>;
|
| 892 |
})}
|
| 893 |
</Fragment>)}</tbody>
|
|
@@ -913,14 +898,6 @@ function SkuGrainList({ grain, panelId, tabId, targets, canEdit, busy, active, o
|
|
| 913 |
</div>;
|
| 914 |
}
|
| 915 |
|
| 916 |
-
function PdpSkuCells({ pdp, target }: { pdp: ProjectPdp; target: Target }) {
|
| 917 |
-
const asin = target.grain === "asin" ? target.key : target.asin;
|
| 918 |
-
const item = pdp.detail?.workflow.items.find(item => item.asin === asin);
|
| 919 |
-
return <><td>{item ? <PdpStep item={item} /> : <span className="lp-proj-muted">Unavailable</span>}</td>
|
| 920 |
-
<td><button type="button" className="lp-proj-btn lp-proj-btn-quiet" disabled={!item || !pdp.open}
|
| 921 |
-
onClick={() => { if (item) pdp.open?.(item); }}>Open</button></td></>;
|
| 922 |
-
}
|
| 923 |
-
|
| 924 |
export function ProjectForm({ targets, onCreated, onCancel, onForbidden }: {
|
| 925 |
targets: Array<{ grain: ProjectGrain; key: string }>; onCreated: (project: Project) => void;
|
| 926 |
onCancel: () => void; onForbidden: () => void;
|
|
|
|
| 11 |
import { useEscapeClose } from "../../ui/useEscapeClose";
|
| 12 |
import ProjectTasks from "./ProjectTasks";
|
| 13 |
import ProjectSkuPicker from "./ProjectSkuPicker";
|
|
|
|
|
|
|
| 14 |
import {
|
| 15 |
EMPTY_FILTER, GRAINS, ProductToolbar, SelectBox, TABLES, normalizeProducts, number, readPage, scopedNodes,
|
| 16 |
type Field, type FilterTree, type Row, type Sorts,
|
|
|
|
| 236 |
}
|
| 237 |
|
| 238 |
export default function ProjectOverview({ id, detail, canEdit: mayEdit, onChanged, onDeleted, onCancel,
|
| 239 |
+
error, setError, setDetail, setTick, formId, active, rowScope = null, onClearRowScope }: {
|
| 240 |
id: string; detail: Detail; canEdit: boolean; onChanged: () => void; onDeleted: () => void; onCancel: () => void;
|
| 241 |
error: string; setError: Dispatch<SetStateAction<string>>; setDetail: Dispatch<SetStateAction<Detail | null>>;
|
| 242 |
setTick: Dispatch<SetStateAction<number>>; formId: string; active: boolean;
|
| 243 |
rowScope?: RowScope | null; onClearRowScope?: () => void;
|
|
|
|
| 244 |
}) {
|
| 245 |
const [draft, setDraft] = useState<OverviewDraft>(() => overviewDraftOf(detail.project));
|
| 246 |
const [people, setPeople] = useState<ProjectPerson[]>([]);
|
|
|
|
| 318 |
setBusy(true);
|
| 319 |
const fallback = "Products were not removed.";
|
| 320 |
try {
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 321 |
const answer = await call<RemoveAnswer>(`/${enc(id)}/targets/remove`,
|
| 322 |
{ method: "POST", body: { targets: pairs } }, fallback);
|
| 323 |
if (!answer?.project || !Array.isArray(answer.targets)) throw new Error("The remove answer is incomplete. Reload the project.");
|
|
|
|
| 343 |
writing.current = false;
|
| 344 |
setBusy(false);
|
| 345 |
}
|
| 346 |
+
}, [detail?.canEdit, id, mayEdit, onChanged, setDetail, setError, setTick]);
|
| 347 |
|
| 348 |
const { project, targets } = detail;
|
| 349 |
+
const canEdit = mayEdit && detail.canEdit;
|
| 350 |
const deleteOpen = confirming && canEdit && active;
|
| 351 |
useEffect(() => {
|
| 352 |
if (!canEdit || !active) { setConfirming(false); setAdding(false); }
|
|
|
|
| 460 |
</section>
|
| 461 |
|
| 462 |
<div className="lp-proj-setup-fields">
|
|
|
|
| 463 |
<SetupRow field="status" label="Status" error={fieldErrors.status} readOnly={!canEdit}>
|
| 464 |
{canEdit
|
| 465 |
? <StatusPicker value={draft.status} disabled={busy} onChange={value => set("status", value)} />
|
|
|
|
| 482 |
onChange={values => { set("ownerUsername", values[0] ?? null); set("owner", null); }} /> : project.owner || project.ownerUsername
|
| 483 |
? <Owner name={project.owner || project.ownerUsername || null} /> : <span className="lp-proj-muted">No owner</span>}
|
| 484 |
</SetupRow>
|
|
|
|
| 485 |
<SetupRow field="contributors" label="Contributors" error={fieldErrors.contributors} readOnly={!canEdit}>
|
| 486 |
<div className="lp-proj-contributors">
|
| 487 |
{contributors.map(person => <span key={person.username} className="lp-proj-contributor">
|
|
|
|
| 555 |
: null}
|
| 556 |
</div>
|
| 557 |
{canEdit && active && adding
|
| 558 |
+
? <ProjectSkuPicker projectId={id} existing={targets} canEdit={canEdit && !busy} onClose={() => setAdding(false)}
|
| 559 |
onAdded={answer => {
|
| 560 |
setDetail(current => current ? { ...current, project: answer.project, targets: answer.targets,
|
| 561 |
cause: answer.cause } : current);
|
|
|
|
| 566 |
: null}
|
| 567 |
{targets.length
|
| 568 |
? <ProjectSkuList targets={targets} scope={scope} canEdit={canEdit} busy={busy} active={active}
|
| 569 |
+
onRemove={removeTargets} />
|
| 570 |
: null}
|
| 571 |
</section>
|
| 572 |
</SetupRow>
|
|
|
|
| 654 |
}
|
| 655 |
}
|
| 656 |
|
| 657 |
+
function ProjectSkuList({ targets: linked, scope, canEdit, busy, active, onRemove }: {
|
| 658 |
targets: Target[]; scope: (RowScope & { clear: () => void }) | null; canEdit: boolean; busy: boolean; active: boolean;
|
| 659 |
onRemove: (pairs: TargetPair[]) => Promise<RemoveOutcome | null>;
|
|
|
|
| 660 |
}) {
|
| 661 |
const id = useId();
|
| 662 |
const [chosen, setChosen] = useState<ProjectGrain>("sku");
|
|
|
|
| 686 |
</div> : null}
|
| 687 |
<SkuGrainList key={grain} grain={grain} panelId={`${id}-panel`} tabId={tabbed ? `${id}-tab-${grain}` : null}
|
| 688 |
targets={targets.filter(target => target.grain === grain)} canEdit={canEdit} busy={busy} active={active}
|
| 689 |
+
onRemove={onRemove} />
|
| 690 |
</div>;
|
| 691 |
}
|
| 692 |
|
| 693 |
+
function SkuGrainList({ grain, panelId, tabId, targets, canEdit, busy, active, onRemove }: {
|
| 694 |
grain: ProjectGrain; panelId: string; tabId: string | null; targets: Target[]; canEdit: boolean;
|
| 695 |
busy: boolean; active: boolean; onRemove: (pairs: TargetPair[]) => Promise<RemoveOutcome | null>;
|
|
|
|
| 696 |
}) {
|
| 697 |
const table = TABLES[grain];
|
| 698 |
const label = GRAIN_LABEL[grain];
|
|
|
|
| 786 |
: new Set(Array.from(current).filter(key => shown.has(key))));
|
| 787 |
}, [term, blocked, shownSignature]);
|
| 788 |
const columns = LIST_COLUMNS[grain];
|
| 789 |
+
const span = (canEdit ? 1 : 0) + 1 + columns.length;
|
| 790 |
const resetView = () => { setSelected(new Set()); setRefused([]); setRemoveError(""); };
|
| 791 |
const clearView = () => { resetView(); setFilters(EMPTY_FILTER); setSorts([]); setGroup(null); setSearch(""); };
|
| 792 |
const viewSet = filtered || sorts.length > 0 || group !== null || selected.size > 0 || search !== "";
|
|
|
|
| 851 |
{canEdit ? <th scope="col" className="lp-proj-skus-check">{boxFor(visibleKeys, `Select all shown ${plural}`)}</th> : null}
|
| 852 |
<th scope="col">Product</th>
|
| 853 |
{columns.map(column => <th scope="col" key={column.key}>{column.label}</th>)}
|
|
|
|
| 854 |
</tr></thead>
|
| 855 |
<tbody>{groups.map(([value, members]) => <Fragment key={value}>
|
| 856 |
{group !== null ? <tr className="lp-proj-sku-group"><th scope="rowgroup" colSpan={span}>
|
|
|
|
| 873 |
return <td key={column.key} className="lp-proj-sku-code" title={shown || undefined}>
|
| 874 |
{shown || <span className="lp-proj-muted">Not set</span>}</td>;
|
| 875 |
})}
|
|
|
|
| 876 |
</tr>;
|
| 877 |
})}
|
| 878 |
</Fragment>)}</tbody>
|
|
|
|
| 898 |
</div>;
|
| 899 |
}
|
| 900 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 901 |
export function ProjectForm({ targets, onCreated, onCancel, onForbidden }: {
|
| 902 |
targets: Array<{ grain: ProjectGrain; key: string }>; onCreated: (project: Project) => void;
|
| 903 |
onCancel: () => void; onForbidden: () => void;
|
web/src/apps/amazon-profit/ProjectSkuPicker.tsx
CHANGED
|
@@ -11,9 +11,6 @@ import { FilterBuilderPanel, Popover, SortBuilder, countConditions,
|
|
| 11 |
import { CenteredPopup, GRAIN_LABEL, ProjectRequestError, call, enc, messageOf, pictureSrc, text,
|
| 12 |
type Project, type ProjectGrain, type Target } from "./ProjectsShared";
|
| 13 |
import "../../styles/sdlc85/project-sku-picker.css";
|
| 14 |
-
import { projectClaimNames, readProjectClaims, type ProjectPdp } from "./pdp/ContinueWorkflow";
|
| 15 |
-
import { addItems, PdpRequestError } from "./pdp/api";
|
| 16 |
-
import type { PdpClaim } from "./pdp/contract";
|
| 17 |
|
| 18 |
export type FilterTree = FilterBuilderPanelProps["filters"];
|
| 19 |
type FilterNode = FilterTree["nodes"][number];
|
|
@@ -175,7 +172,7 @@ export type TaskTargetSource = {
|
|
| 175 |
name: string; targets?: ReadonlyArray<{ grain: string; key: string }> | null;
|
| 176 |
targetFilter?: { grain: string; filter: unknown } | null; completionFilter?: unknown;
|
| 177 |
};
|
| 178 |
-
type PickerBase = { projectId: string; canEdit: boolean; onClose: () => void
|
| 179 |
export type SkuPickerProps = PickerBase & (
|
| 180 |
| { mode: "add"; existing: Target[]; onAdded: (answer: ProjectSkuAddAnswer) => void }
|
| 181 |
| { mode: "task"; task: TaskTargetSource; onAssign: (change: TaskTargetChange) => Promise<void> });
|
|
@@ -204,12 +201,11 @@ function taskStart(task: TaskTargetSource | null) {
|
|
| 204 |
done: treeOf(task?.completionFilter), picked, assigned: !!saved || Object.keys(picked).length > 0 };
|
| 205 |
}
|
| 206 |
|
| 207 |
-
export default function ProjectSkuPicker({ projectId, existing, canEdit, onAdded, onClose
|
| 208 |
projectId: string; existing: Target[]; canEdit: boolean;
|
| 209 |
onAdded: (answer: ProjectSkuAddAnswer) => void; onClose: () => void;
|
| 210 |
-
pdp?: ProjectPdp;
|
| 211 |
}) {
|
| 212 |
-
return <SkuPicker mode="add" projectId={projectId} existing={existing} canEdit={canEdit} onAdded={onAdded} onClose={onClose}
|
| 213 |
}
|
| 214 |
|
| 215 |
/** One picker body: `add` links products to the project, `task` sets one task's targets and completion filter. */
|
|
@@ -217,15 +213,11 @@ export function SkuPicker(props: SkuPickerProps) {
|
|
| 217 |
const { projectId, canEdit, onClose } = props;
|
| 218 |
const task = props.mode === "task" ? props : null;
|
| 219 |
const taskMode = task !== null;
|
| 220 |
-
const pdp = props.mode === "add" ? props.pdp : undefined;
|
| 221 |
-
const pdpMode = Boolean(pdp);
|
| 222 |
-
const workflow = pdp?.detail?.workflow;
|
| 223 |
-
const [taken, setTaken] = useState<PdpClaim[]>([]);
|
| 224 |
const existing = props.mode === "add" ? props.existing : NO_TARGETS;
|
| 225 |
const onAdded = props.mode === "add" ? props.onAdded : ignoreAnswer;
|
| 226 |
const id = useId();
|
| 227 |
const [start] = useState(() => taskStart(task?.task ?? null));
|
| 228 |
-
const [grain, setGrain] = useState<ProjectGrain>(
|
| 229 |
const [query, setQuery] = useState("");
|
| 230 |
const [search, setSearch] = useState("");
|
| 231 |
const [filters, setFilters] = useState<FilterTree>(start.filter);
|
|
@@ -269,17 +261,15 @@ export function SkuPicker(props: SkuPickerProps) {
|
|
| 269 |
const order = useMemo<Sorts>(() => group
|
| 270 |
? [{ colId: group, dir: "asc" }, ...sorts.filter(sort => sort.colId !== group)] : sorts, [group, sorts]);
|
| 271 |
const available = Math.max(0, (match?.total ?? 0) - (match?.linked ?? 0));
|
| 272 |
-
const
|
| 273 |
-
const
|
| 274 |
-
: workflow ? Array.from(selected).filter(key => !holder(key)).length : selected.size;
|
| 275 |
-
const ready = canEdit && (!pdp || Boolean(pdp.detail?.canEdit)) && !forbidden && !loading && !busy && query === search && match !== null && !error;
|
| 276 |
const isSelected = (key: string) => linked.has(key) || (allMatching ? !excluded.has(key) : selected.has(key));
|
| 277 |
-
const pageKeys = rows.map(row => text(row[grain])).filter(key => key && !linked.has(key)
|
| 278 |
const checkedOnPage = pageKeys.filter(isSelected).length;
|
| 279 |
const preview = taskMode && kind === "filter";
|
| 280 |
const picks = selected.size + Object.entries(held)
|
| 281 |
.reduce((sum, [value, keys]) => value === grain ? sum : sum + (keys?.length ?? 0), 0);
|
| 282 |
-
const grains =
|
| 283 |
|
| 284 |
const resetSelection = () => { setSelected(new Set()); setAllMatching(false); setExcluded(new Set()); };
|
| 285 |
const resetRead = () => {
|
|
@@ -335,11 +325,6 @@ export function SkuPicker(props: SkuPickerProps) {
|
|
| 335 |
if (counted && counted.total > 0 && !products.length) {
|
| 336 |
throw new Error("Matching products did not load. Retry to refresh the matches.");
|
| 337 |
}
|
| 338 |
-
if (pdpMode) {
|
| 339 |
-
const claims = await readProjectClaims(products.map(row => text(row.asin)), controller.signal);
|
| 340 |
-
if (controller.signal.aborted) return;
|
| 341 |
-
setTaken(current => cursor === null ? claims : [...current, ...claims]);
|
| 342 |
-
}
|
| 343 |
setFields(page.fields);
|
| 344 |
setRows(current => {
|
| 345 |
if (counted?.total === 0) return [];
|
|
@@ -370,7 +355,7 @@ export function SkuPicker(props: SkuPickerProps) {
|
|
| 370 |
})
|
| 371 |
.finally(() => { if (!controller.signal.aborted) setLoading(false); });
|
| 372 |
return () => controller.abort();
|
| 373 |
-
}, [projectId, canEdit, grain, table, tree, order, cursor, attempt, linkedSignature, taskMode
|
| 374 |
|
| 375 |
const toggleKeys = (keys: string[], checked: boolean) => {
|
| 376 |
if (allMatching) setExcluded(current => {
|
|
@@ -389,11 +374,6 @@ export function SkuPicker(props: SkuPickerProps) {
|
|
| 389 |
if (!ready || !selectedCount || submitting.current) return;
|
| 390 |
submitting.current = true; setBusy(true); setRefused([]);
|
| 391 |
try {
|
| 392 |
-
if (pdp) {
|
| 393 |
-
pdp.accept((await addItems(projectId, Array.from(selected).filter(key => !holder(key)))).workflow);
|
| 394 |
-
onClose();
|
| 395 |
-
return;
|
| 396 |
-
}
|
| 397 |
// Exclusions apply after server ranking, so they cannot promote a different top-N row.
|
| 398 |
const complete = allMatching && excluded.size
|
| 399 |
? await matchingKeys(projectId, grain, tree, match!.total) : null;
|
|
@@ -413,11 +393,7 @@ export function SkuPicker(props: SkuPickerProps) {
|
|
| 413 |
} else onClose();
|
| 414 |
} catch (reason) {
|
| 415 |
setError(messageOf(reason, "Products were not added."));
|
| 416 |
-
if (reason instanceof
|
| 417 |
-
try { setTaken(await readProjectClaims(rows.map(row => text(row.asin)))); }
|
| 418 |
-
catch (refreshReason) { setError(messageOf(refreshReason, "Workflow claims did not load. Retry.")); }
|
| 419 |
-
}
|
| 420 |
-
if ((reason instanceof ProjectRequestError || reason instanceof PdpRequestError) && [401, 403].includes(reason.status)) {
|
| 421 |
setForbidden(true); setRows([]); setFields([]); setMatch(null);
|
| 422 |
}
|
| 423 |
} finally { submitting.current = false; setBusy(false); }
|
|
@@ -438,10 +414,7 @@ export function SkuPicker(props: SkuPickerProps) {
|
|
| 438 |
try {
|
| 439 |
const every = await matchingKeys(projectId, grain, tree, current.total);
|
| 440 |
if (every.total !== current.total) throw new Error("Products changed. Retry to refresh the matches.");
|
| 441 |
-
|
| 442 |
-
if (workflow) setTaken(claims);
|
| 443 |
-
const eligible = workflow ? every.keys.filter(key => !linked.has(key) && !projectClaimNames(claims, workflow, key)) : every.keys;
|
| 444 |
-
setSelected(previous => new Set([...previous, ...eligible]));
|
| 445 |
} catch (reason) { refuse(reason, "Matches did not load."); }
|
| 446 |
finally { submitting.current = false; setBusy(false); }
|
| 447 |
};
|
|
@@ -535,7 +508,7 @@ export function SkuPicker(props: SkuPickerProps) {
|
|
| 535 |
{match?.linked ? ` · ${number(match.linked)} already linked` : ""}</span>
|
| 536 |
{match && available > 0 && !preview ? <button type="button" className="lp-proj-sku-link" disabled={!ready}
|
| 537 |
onClick={() => {
|
| 538 |
-
if (taskMode
|
| 539 |
setAllMatching(true); setSelected(new Set()); setExcluded(new Set());
|
| 540 |
}}>
|
| 541 |
Select all {number(match.total)} matching
|
|
@@ -570,15 +543,13 @@ export function SkuPicker(props: SkuPickerProps) {
|
|
| 570 |
{members.map(row => {
|
| 571 |
const key = text(row[grain]);
|
| 572 |
const alreadyLinked = linked.has(key);
|
| 573 |
-
const heldBy = holder(key);
|
| 574 |
const picture = pictureSrc(text(row.picture));
|
| 575 |
return <tr key={key} data-selected={!preview && isSelected(key) || undefined}>
|
| 576 |
-
{preview ? null : <td><SelectBox checked={isSelected(key)
|
| 577 |
label={`${alreadyLinked ? "Already linked" : "Select"} ${key}`} onChange={checked => toggleKeys([key], checked)} /></td>}
|
| 578 |
<td><div className="lp-proj-sku-product">
|
| 579 |
{picture ? <img src={picture} alt="" loading="lazy" /> : <span className="lp-proj-sku-picture" aria-hidden="true" />}
|
| 580 |
-
<span><span>{text(row.title) || key}</span>{alreadyLinked ? <small>Already linked</small> : null}
|
| 581 |
-
{heldBy ? <small>{heldBy} already changes the same parts</small> : null}</span>
|
| 582 |
</div></td>
|
| 583 |
<td className="lp-proj-sku-code" title={key}>{key}</td>
|
| 584 |
<td className="lp-proj-sku-code">{grain === "parent_asin"
|
|
|
|
| 11 |
import { CenteredPopup, GRAIN_LABEL, ProjectRequestError, call, enc, messageOf, pictureSrc, text,
|
| 12 |
type Project, type ProjectGrain, type Target } from "./ProjectsShared";
|
| 13 |
import "../../styles/sdlc85/project-sku-picker.css";
|
|
|
|
|
|
|
|
|
|
| 14 |
|
| 15 |
export type FilterTree = FilterBuilderPanelProps["filters"];
|
| 16 |
type FilterNode = FilterTree["nodes"][number];
|
|
|
|
| 172 |
name: string; targets?: ReadonlyArray<{ grain: string; key: string }> | null;
|
| 173 |
targetFilter?: { grain: string; filter: unknown } | null; completionFilter?: unknown;
|
| 174 |
};
|
| 175 |
+
type PickerBase = { projectId: string; canEdit: boolean; onClose: () => void };
|
| 176 |
export type SkuPickerProps = PickerBase & (
|
| 177 |
| { mode: "add"; existing: Target[]; onAdded: (answer: ProjectSkuAddAnswer) => void }
|
| 178 |
| { mode: "task"; task: TaskTargetSource; onAssign: (change: TaskTargetChange) => Promise<void> });
|
|
|
|
| 201 |
done: treeOf(task?.completionFilter), picked, assigned: !!saved || Object.keys(picked).length > 0 };
|
| 202 |
}
|
| 203 |
|
| 204 |
+
export default function ProjectSkuPicker({ projectId, existing, canEdit, onAdded, onClose }: {
|
| 205 |
projectId: string; existing: Target[]; canEdit: boolean;
|
| 206 |
onAdded: (answer: ProjectSkuAddAnswer) => void; onClose: () => void;
|
|
|
|
| 207 |
}) {
|
| 208 |
+
return <SkuPicker mode="add" projectId={projectId} existing={existing} canEdit={canEdit} onAdded={onAdded} onClose={onClose} />;
|
| 209 |
}
|
| 210 |
|
| 211 |
/** One picker body: `add` links products to the project, `task` sets one task's targets and completion filter. */
|
|
|
|
| 213 |
const { projectId, canEdit, onClose } = props;
|
| 214 |
const task = props.mode === "task" ? props : null;
|
| 215 |
const taskMode = task !== null;
|
|
|
|
|
|
|
|
|
|
|
|
|
| 216 |
const existing = props.mode === "add" ? props.existing : NO_TARGETS;
|
| 217 |
const onAdded = props.mode === "add" ? props.onAdded : ignoreAnswer;
|
| 218 |
const id = useId();
|
| 219 |
const [start] = useState(() => taskStart(task?.task ?? null));
|
| 220 |
+
const [grain, setGrain] = useState<ProjectGrain>(start.grain);
|
| 221 |
const [query, setQuery] = useState("");
|
| 222 |
const [search, setSearch] = useState("");
|
| 223 |
const [filters, setFilters] = useState<FilterTree>(start.filter);
|
|
|
|
| 261 |
const order = useMemo<Sorts>(() => group
|
| 262 |
? [{ colId: group, dir: "asc" }, ...sorts.filter(sort => sort.colId !== group)] : sorts, [group, sorts]);
|
| 263 |
const available = Math.max(0, (match?.total ?? 0) - (match?.linked ?? 0));
|
| 264 |
+
const selectedCount = allMatching ? Math.max(0, available - excluded.size) : selected.size;
|
| 265 |
+
const ready = canEdit && !forbidden && !loading && !busy && query === search && match !== null && !error;
|
|
|
|
|
|
|
| 266 |
const isSelected = (key: string) => linked.has(key) || (allMatching ? !excluded.has(key) : selected.has(key));
|
| 267 |
+
const pageKeys = rows.map(row => text(row[grain])).filter(key => key && !linked.has(key));
|
| 268 |
const checkedOnPage = pageKeys.filter(isSelected).length;
|
| 269 |
const preview = taskMode && kind === "filter";
|
| 270 |
const picks = selected.size + Object.entries(held)
|
| 271 |
.reduce((sum, [value, keys]) => value === grain ? sum : sum + (keys?.length ?? 0), 0);
|
| 272 |
+
const grains = taskMode ? TASK_GRAINS : GRAINS;
|
| 273 |
|
| 274 |
const resetSelection = () => { setSelected(new Set()); setAllMatching(false); setExcluded(new Set()); };
|
| 275 |
const resetRead = () => {
|
|
|
|
| 325 |
if (counted && counted.total > 0 && !products.length) {
|
| 326 |
throw new Error("Matching products did not load. Retry to refresh the matches.");
|
| 327 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 328 |
setFields(page.fields);
|
| 329 |
setRows(current => {
|
| 330 |
if (counted?.total === 0) return [];
|
|
|
|
| 355 |
})
|
| 356 |
.finally(() => { if (!controller.signal.aborted) setLoading(false); });
|
| 357 |
return () => controller.abort();
|
| 358 |
+
}, [projectId, canEdit, grain, table, tree, order, cursor, attempt, linkedSignature, taskMode]);
|
| 359 |
|
| 360 |
const toggleKeys = (keys: string[], checked: boolean) => {
|
| 361 |
if (allMatching) setExcluded(current => {
|
|
|
|
| 374 |
if (!ready || !selectedCount || submitting.current) return;
|
| 375 |
submitting.current = true; setBusy(true); setRefused([]);
|
| 376 |
try {
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 377 |
// Exclusions apply after server ranking, so they cannot promote a different top-N row.
|
| 378 |
const complete = allMatching && excluded.size
|
| 379 |
? await matchingKeys(projectId, grain, tree, match!.total) : null;
|
|
|
|
| 393 |
} else onClose();
|
| 394 |
} catch (reason) {
|
| 395 |
setError(messageOf(reason, "Products were not added."));
|
| 396 |
+
if (reason instanceof ProjectRequestError && [401, 403].includes(reason.status)) {
|
|
|
|
|
|
|
|
|
|
|
|
|
| 397 |
setForbidden(true); setRows([]); setFields([]); setMatch(null);
|
| 398 |
}
|
| 399 |
} finally { submitting.current = false; setBusy(false); }
|
|
|
|
| 414 |
try {
|
| 415 |
const every = await matchingKeys(projectId, grain, tree, current.total);
|
| 416 |
if (every.total !== current.total) throw new Error("Products changed. Retry to refresh the matches.");
|
| 417 |
+
setSelected(previous => new Set([...previous, ...every.keys]));
|
|
|
|
|
|
|
|
|
|
| 418 |
} catch (reason) { refuse(reason, "Matches did not load."); }
|
| 419 |
finally { submitting.current = false; setBusy(false); }
|
| 420 |
};
|
|
|
|
| 508 |
{match?.linked ? ` · ${number(match.linked)} already linked` : ""}</span>
|
| 509 |
{match && available > 0 && !preview ? <button type="button" className="lp-proj-sku-link" disabled={!ready}
|
| 510 |
onClick={() => {
|
| 511 |
+
if (taskMode) { void selectMatching(); return; }
|
| 512 |
setAllMatching(true); setSelected(new Set()); setExcluded(new Set());
|
| 513 |
}}>
|
| 514 |
Select all {number(match.total)} matching
|
|
|
|
| 543 |
{members.map(row => {
|
| 544 |
const key = text(row[grain]);
|
| 545 |
const alreadyLinked = linked.has(key);
|
|
|
|
| 546 |
const picture = pictureSrc(text(row.picture));
|
| 547 |
return <tr key={key} data-selected={!preview && isSelected(key) || undefined}>
|
| 548 |
+
{preview ? null : <td><SelectBox checked={isSelected(key)} disabled={!ready || alreadyLinked}
|
| 549 |
label={`${alreadyLinked ? "Already linked" : "Select"} ${key}`} onChange={checked => toggleKeys([key], checked)} /></td>}
|
| 550 |
<td><div className="lp-proj-sku-product">
|
| 551 |
{picture ? <img src={picture} alt="" loading="lazy" /> : <span className="lp-proj-sku-picture" aria-hidden="true" />}
|
| 552 |
+
<span><span>{text(row.title) || key}</span>{alreadyLinked ? <small>Already linked</small> : null}</span>
|
|
|
|
| 553 |
</div></td>
|
| 554 |
<td className="lp-proj-sku-code" title={key}>{key}</td>
|
| 555 |
<td className="lp-proj-sku-code">{grain === "parent_asin"
|
web/src/apps/amazon-profit/Projects.tsx
CHANGED
|
@@ -1,16 +1,12 @@
|
|
| 1 |
-
import { useCallback, useEffect, useId, useMemo, useRef, useState
|
| 2 |
import {
|
| 3 |
-
CenteredPopup, GRAIN_LABEL, LoadingRows,
|
| 4 |
type Detail, type ProjectGrain, type ProjectList,
|
| 5 |
} from "./ProjectsShared";
|
| 6 |
import ProjectResults from "./ProjectResults";
|
| 7 |
import { AddExisting, ProjectBrowser } from "./ProjectsList";
|
| 8 |
-
import ProjectOverview, { ProjectForm, RowScopeChip,
|
| 9 |
import ProjectActivities from "./ProjectActivities";
|
| 10 |
-
import PdpActivities from "./pdp/PdpActivities";
|
| 11 |
-
import BrandRules from "./pdp/BrandRules";
|
| 12 |
-
import ContinueWorkflow, { useProjectPdp, type PdpProjectNavigation } from "./pdp/ContinueWorkflow";
|
| 13 |
-
import type { PdpProjectPane } from "./pdp/contract";
|
| 14 |
|
| 15 |
export { CenteredPopup, projectsTargetFromRow } from "./ProjectsShared";
|
| 16 |
export type { CenteredPopupProps, Project, ProjectGrain, ProjectsTarget } from "./ProjectsShared";
|
|
@@ -23,29 +19,16 @@ const PROJECT_PANES = [
|
|
| 23 |
{ key: "results", label: "Dashboard" },
|
| 24 |
{ key: "log", label: "Activities" },
|
| 25 |
] as const;
|
| 26 |
-
type
|
| 27 |
-
type ProjectExtras = {
|
| 28 |
-
pdp?: PdpProjectNavigation; backLabel?: string; onBack?: () => void;
|
| 29 |
-
initialPane?: PdpProjectPane;
|
| 30 |
-
headerExtra?: ReactNode; extraPanes?: ExtraPane[];
|
| 31 |
-
};
|
| 32 |
const RESULT_GRAIN = { sku: "sku", asin: "asin", parent_asin: "parent" } as const;
|
| 33 |
|
| 34 |
-
function ProjectDetail({ id, canEdit: mayEdit, rowScope: openedFrom, onChanged, onDeleted, onCancel
|
| 35 |
-
pdp: navigation, headerExtra, extraPanes, initialPane }: {
|
| 36 |
id: string; canEdit: boolean; rowScope?: RowScope; onChanged: () => void; onDeleted: () => void; onCancel: () => void;
|
| 37 |
-
}
|
| 38 |
const [detail, setDetail] = useState<Detail | null>(null);
|
| 39 |
const [error, setError] = useState("");
|
| 40 |
const [tick, setTick] = useState(0);
|
| 41 |
-
const [pane, setPane] = useState<
|
| 42 |
-
const projectChanged = () => { setTick(value => value + 1); onChanged(); };
|
| 43 |
-
const pdp = useProjectPdp(detail?.project, detail?.targets ?? [], tick, mayEdit && Boolean(detail?.canEdit), projectChanged, navigation);
|
| 44 |
-
const extras = detail?.project.type === "pdp" ? [
|
| 45 |
-
{ key: "brand", label: "Brand and rules", content: <BrandRules key={id} workflowId={id} projectName={detail.project.name}
|
| 46 |
-
canEdit={mayEdit && detail.canEdit} active={pane === "brand"} /> }, ...(extraPanes ?? []),
|
| 47 |
-
] : [];
|
| 48 |
-
const panes = extras.length ? [...PROJECT_PANES, ...extras] : PROJECT_PANES;
|
| 49 |
// The row a Projects cell opened from scopes the Overview SKU list and the Dashboard until it is cleared.
|
| 50 |
const [rowScope, setRowScope] = useState<RowScope | null>(() => openedFrom?.key.trim()
|
| 51 |
? { grain: openedFrom.grain, key: openedFrom.grain === "sku" ? openedFrom.key.trim() : openedFrom.key.trim().toUpperCase() }
|
|
@@ -81,20 +64,20 @@ function ProjectDetail({ id, canEdit: mayEdit, rowScope: openedFrom, onChanged,
|
|
| 81 |
|
| 82 |
const rail = <nav className="lp-proj-detail-rail" ref={railRef} aria-label="Project sections">
|
| 83 |
<div className="lp-proj-detail-rail-list" role="tablist" aria-label="Project sections" aria-orientation="vertical">
|
| 84 |
-
{
|
| 85 |
className={`lp-proj-detail-rail-row${pane === key ? " is-on" : ""}`}
|
| 86 |
id={`${formId}-tab-${key}`} aria-controls={`${formId}-pane-${key}`}
|
| 87 |
aria-selected={pane === key} tabIndex={pane === key ? 0 : -1} onClick={() => setPane(key)}
|
| 88 |
onKeyDown={event => {
|
| 89 |
let next: number;
|
| 90 |
-
if (event.key === "ArrowDown" || event.key === "ArrowRight") next = (index + 1) %
|
| 91 |
-
else if (event.key === "ArrowUp" || event.key === "ArrowLeft") next = (index - 1 +
|
| 92 |
else if (event.key === "Home") next = 0;
|
| 93 |
-
else if (event.key === "End") next =
|
| 94 |
else return;
|
| 95 |
event.preventDefault();
|
| 96 |
event.stopPropagation();
|
| 97 |
-
setPane(
|
| 98 |
railRef.current?.querySelectorAll<HTMLButtonElement>('[role="tab"]')[next]?.focus();
|
| 99 |
}}><span className="lp-proj-detail-rail-label">{label}</span></button>)}
|
| 100 |
</div>
|
|
@@ -114,19 +97,15 @@ function ProjectDetail({ id, canEdit: mayEdit, rowScope: openedFrom, onChanged,
|
|
| 114 |
</div>;
|
| 115 |
const { project } = detail;
|
| 116 |
const canEdit = mayEdit && detail.canEdit;
|
| 117 |
-
const scopedAsins = rowScope ? [...new Set(detail.targets.filter(target => inRowScope(target, rowScope))
|
| 118 |
-
.flatMap(target => target.grain === "asin" ? [target.key] : target.asin ? [target.asin] : []))] : [];
|
| 119 |
|
| 120 |
return <div className="lp-proj-detail-layout">
|
| 121 |
-
<ProjectHeader>{pdp ? <ContinueWorkflow pdp={pdp} targets={detail.targets}
|
| 122 |
-
asin={scopedAsins.length === 1 ? scopedAsins[0] : undefined} /> : null}{headerExtra}</ProjectHeader>
|
| 123 |
{rail}
|
| 124 |
<div className="lp-proj-panes" ref={panesRef}>
|
| 125 |
<section className="lp-proj-pane" role="tabpanel" id={`${formId}-pane-setup`}
|
| 126 |
aria-labelledby={`${formId}-tab-setup`} hidden={pane !== "setup"} tabIndex={0}>
|
| 127 |
<ProjectOverview id={id} detail={detail} canEdit={mayEdit} onChanged={onChanged} onDeleted={onDeleted}
|
| 128 |
onCancel={onCancel} error={error} setError={setError} setDetail={setDetail} setTick={setTick}
|
| 129 |
-
formId={formId} active={pane === "setup"} rowScope={rowScope} onClearRowScope={clearRowScope}
|
| 130 |
</section>
|
| 131 |
<section className="lp-proj-pane lp-proj-pane-stack" role="tabpanel" id={`${formId}-pane-results`}
|
| 132 |
aria-labelledby={`${formId}-tab-results`} hidden={pane !== "results"} tabIndex={0}>
|
|
@@ -135,24 +114,17 @@ function ProjectDetail({ id, canEdit: mayEdit, rowScope: openedFrom, onChanged,
|
|
| 135 |
<ProjectResults projectId={id} configs={project.resultConfigs ?? []} completedOn={project.completedOn}
|
| 136 |
updatedAt={project.updatedAt} scope={resultScope} canEdit={canEdit} onSaved={() => { setTick(value => value + 1); onChanged(); }} />
|
| 137 |
</section>
|
| 138 |
-
{project.
|
| 139 |
-
? <PdpActivities project={project} formId={formId} active={pane === "log"} />
|
| 140 |
-
: <ProjectActivities project={project} entries={detail.activity} formId={formId} active={pane === "log"} />}
|
| 141 |
-
{extras.map(extra => <section key={extra.key} className="lp-proj-pane" role="tabpanel" tabIndex={0}
|
| 142 |
-
id={`${formId}-pane-${extra.key}`} aria-labelledby={`${formId}-tab-${extra.key}`} hidden={pane !== extra.key}>
|
| 143 |
-
{extra.content}
|
| 144 |
-
</section>)}
|
| 145 |
</div>
|
| 146 |
</div>;
|
| 147 |
}
|
| 148 |
|
| 149 |
// ---- The popup a Projects cell opens ----
|
| 150 |
|
| 151 |
-
export interface ProjectsPopupProps
|
| 152 |
grain: ProjectGrain;
|
| 153 |
rowKey: string;
|
| 154 |
title?: string;
|
| 155 |
-
selectedProjectId?: string;
|
| 156 |
onClose: () => void;
|
| 157 |
onChanged?: () => void;
|
| 158 |
}
|
|
@@ -160,13 +132,11 @@ export interface ProjectsPopupProps extends ProjectExtras {
|
|
| 160 |
type PopupView = { kind: "list" } | { kind: "detail"; id: string } | { kind: "new" } | { kind: "add" };
|
| 161 |
|
| 162 |
export function ProjectsPopup(props: ProjectsPopupProps) {
|
| 163 |
-
return <ProjectsPopupContent key={`${props.grain}:${props.rowKey}
|
| 164 |
}
|
| 165 |
|
| 166 |
-
function ProjectsPopupContent({ grain, rowKey, title,
|
| 167 |
-
|
| 168 |
-
const [view, setView] = useState<PopupView>(() => selectedProjectId
|
| 169 |
-
? { kind: "detail", id: selectedProjectId } : { kind: "list" });
|
| 170 |
const [read, setRead] = useState<ProjectList | null>(null);
|
| 171 |
const [error, setError] = useState("");
|
| 172 |
const [tick, setTick] = useState(0);
|
|
@@ -186,7 +156,7 @@ function ProjectsPopupContent({ grain, rowKey, title, selectedProjectId, onClose
|
|
| 186 |
const label = `${GRAIN_LABEL[grain]} ${rowKey}`;
|
| 187 |
|
| 188 |
const actions = view.kind === "detail"
|
| 189 |
-
? <button type="button" className="lp-proj-btn lp-proj-btn-quiet" onClick={
|
| 190 |
: view.kind === "list" && canEdit
|
| 191 |
? <>
|
| 192 |
<button type="button" className="lp-proj-btn lp-proj-btn-quiet" onClick={() => setView({ kind: "add" })}>Add existing project</button>
|
|
@@ -213,8 +183,6 @@ function ProjectsPopupContent({ grain, rowKey, title, selectedProjectId, onClose
|
|
| 213 |
: null}
|
| 214 |
{view.kind === "detail"
|
| 215 |
? <ProjectDetail key={view.id} id={view.id} canEdit={canEdit} rowScope={{ grain, key: rowKey }}
|
| 216 |
-
initialPane={view.id === selectedProjectId ? initialPane : undefined}
|
| 217 |
-
pdp={pdp} backLabel={backLabel} onBack={onBack} headerExtra={headerExtra} extraPanes={extraPanes}
|
| 218 |
onChanged={changed} onDeleted={onClose} onCancel={() => setView({ kind: "list" })} />
|
| 219 |
: null}
|
| 220 |
{view.kind === "new" && canEdit
|
|
@@ -236,7 +204,7 @@ function ProjectsPopupContent({ grain, rowKey, title, selectedProjectId, onClose
|
|
| 236 |
|
| 237 |
// ---- The Projects tab: every project, the same rows ----
|
| 238 |
|
| 239 |
-
export function ProjectsPage(
|
| 240 |
const [read, setRead] = useState<ProjectList | null>(null);
|
| 241 |
const [error, setError] = useState("");
|
| 242 |
const [tick, setTick] = useState(0);
|
|
@@ -273,7 +241,6 @@ export function ProjectsPage({ pdp, headerExtra, extraPanes }: ProjectExtras = {
|
|
| 273 |
{open
|
| 274 |
? <CenteredPopup className="lp-proj-popup" title="Project" onClose={() => setOpen(null)}>
|
| 275 |
<ProjectDetail key={open} id={open} canEdit={Boolean(read?.canEdit)}
|
| 276 |
-
pdp={pdp} headerExtra={headerExtra} extraPanes={extraPanes}
|
| 277 |
onChanged={reload} onDeleted={() => { setOpen(null); reload(); }} onCancel={() => setOpen(null)} />
|
| 278 |
</CenteredPopup>
|
| 279 |
: null}
|
|
|
|
| 1 |
+
import { useCallback, useEffect, useId, useMemo, useRef, useState } from "react";
|
| 2 |
import {
|
| 3 |
+
CenteredPopup, GRAIN_LABEL, LoadingRows, call, enc, messageOf,
|
| 4 |
type Detail, type ProjectGrain, type ProjectList,
|
| 5 |
} from "./ProjectsShared";
|
| 6 |
import ProjectResults from "./ProjectResults";
|
| 7 |
import { AddExisting, ProjectBrowser } from "./ProjectsList";
|
| 8 |
+
import ProjectOverview, { ProjectForm, RowScopeChip, type RowScope } from "./ProjectOverview";
|
| 9 |
import ProjectActivities from "./ProjectActivities";
|
|
|
|
|
|
|
|
|
|
|
|
|
| 10 |
|
| 11 |
export { CenteredPopup, projectsTargetFromRow } from "./ProjectsShared";
|
| 12 |
export type { CenteredPopupProps, Project, ProjectGrain, ProjectsTarget } from "./ProjectsShared";
|
|
|
|
| 19 |
{ key: "results", label: "Dashboard" },
|
| 20 |
{ key: "log", label: "Activities" },
|
| 21 |
] as const;
|
| 22 |
+
type ProjectPane = (typeof PROJECT_PANES)[number]["key"];
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 23 |
const RESULT_GRAIN = { sku: "sku", asin: "asin", parent_asin: "parent" } as const;
|
| 24 |
|
| 25 |
+
function ProjectDetail({ id, canEdit: mayEdit, rowScope: openedFrom, onChanged, onDeleted, onCancel }: {
|
|
|
|
| 26 |
id: string; canEdit: boolean; rowScope?: RowScope; onChanged: () => void; onDeleted: () => void; onCancel: () => void;
|
| 27 |
+
}) {
|
| 28 |
const [detail, setDetail] = useState<Detail | null>(null);
|
| 29 |
const [error, setError] = useState("");
|
| 30 |
const [tick, setTick] = useState(0);
|
| 31 |
+
const [pane, setPane] = useState<ProjectPane>("setup");
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 32 |
// The row a Projects cell opened from scopes the Overview SKU list and the Dashboard until it is cleared.
|
| 33 |
const [rowScope, setRowScope] = useState<RowScope | null>(() => openedFrom?.key.trim()
|
| 34 |
? { grain: openedFrom.grain, key: openedFrom.grain === "sku" ? openedFrom.key.trim() : openedFrom.key.trim().toUpperCase() }
|
|
|
|
| 64 |
|
| 65 |
const rail = <nav className="lp-proj-detail-rail" ref={railRef} aria-label="Project sections">
|
| 66 |
<div className="lp-proj-detail-rail-list" role="tablist" aria-label="Project sections" aria-orientation="vertical">
|
| 67 |
+
{PROJECT_PANES.map(({ key, label }, index) => <button key={key} type="button" role="tab"
|
| 68 |
className={`lp-proj-detail-rail-row${pane === key ? " is-on" : ""}`}
|
| 69 |
id={`${formId}-tab-${key}`} aria-controls={`${formId}-pane-${key}`}
|
| 70 |
aria-selected={pane === key} tabIndex={pane === key ? 0 : -1} onClick={() => setPane(key)}
|
| 71 |
onKeyDown={event => {
|
| 72 |
let next: number;
|
| 73 |
+
if (event.key === "ArrowDown" || event.key === "ArrowRight") next = (index + 1) % PROJECT_PANES.length;
|
| 74 |
+
else if (event.key === "ArrowUp" || event.key === "ArrowLeft") next = (index - 1 + PROJECT_PANES.length) % PROJECT_PANES.length;
|
| 75 |
else if (event.key === "Home") next = 0;
|
| 76 |
+
else if (event.key === "End") next = PROJECT_PANES.length - 1;
|
| 77 |
else return;
|
| 78 |
event.preventDefault();
|
| 79 |
event.stopPropagation();
|
| 80 |
+
setPane(PROJECT_PANES[next].key);
|
| 81 |
railRef.current?.querySelectorAll<HTMLButtonElement>('[role="tab"]')[next]?.focus();
|
| 82 |
}}><span className="lp-proj-detail-rail-label">{label}</span></button>)}
|
| 83 |
</div>
|
|
|
|
| 97 |
</div>;
|
| 98 |
const { project } = detail;
|
| 99 |
const canEdit = mayEdit && detail.canEdit;
|
|
|
|
|
|
|
| 100 |
|
| 101 |
return <div className="lp-proj-detail-layout">
|
|
|
|
|
|
|
| 102 |
{rail}
|
| 103 |
<div className="lp-proj-panes" ref={panesRef}>
|
| 104 |
<section className="lp-proj-pane" role="tabpanel" id={`${formId}-pane-setup`}
|
| 105 |
aria-labelledby={`${formId}-tab-setup`} hidden={pane !== "setup"} tabIndex={0}>
|
| 106 |
<ProjectOverview id={id} detail={detail} canEdit={mayEdit} onChanged={onChanged} onDeleted={onDeleted}
|
| 107 |
onCancel={onCancel} error={error} setError={setError} setDetail={setDetail} setTick={setTick}
|
| 108 |
+
formId={formId} active={pane === "setup"} rowScope={rowScope} onClearRowScope={clearRowScope} />
|
| 109 |
</section>
|
| 110 |
<section className="lp-proj-pane lp-proj-pane-stack" role="tabpanel" id={`${formId}-pane-results`}
|
| 111 |
aria-labelledby={`${formId}-tab-results`} hidden={pane !== "results"} tabIndex={0}>
|
|
|
|
| 114 |
<ProjectResults projectId={id} configs={project.resultConfigs ?? []} completedOn={project.completedOn}
|
| 115 |
updatedAt={project.updatedAt} scope={resultScope} canEdit={canEdit} onSaved={() => { setTick(value => value + 1); onChanged(); }} />
|
| 116 |
</section>
|
| 117 |
+
<ProjectActivities project={project} entries={detail.activity} formId={formId} active={pane === "log"} />
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 118 |
</div>
|
| 119 |
</div>;
|
| 120 |
}
|
| 121 |
|
| 122 |
// ---- The popup a Projects cell opens ----
|
| 123 |
|
| 124 |
+
export interface ProjectsPopupProps {
|
| 125 |
grain: ProjectGrain;
|
| 126 |
rowKey: string;
|
| 127 |
title?: string;
|
|
|
|
| 128 |
onClose: () => void;
|
| 129 |
onChanged?: () => void;
|
| 130 |
}
|
|
|
|
| 132 |
type PopupView = { kind: "list" } | { kind: "detail"; id: string } | { kind: "new" } | { kind: "add" };
|
| 133 |
|
| 134 |
export function ProjectsPopup(props: ProjectsPopupProps) {
|
| 135 |
+
return <ProjectsPopupContent key={`${props.grain}:${props.rowKey}`} {...props} />;
|
| 136 |
}
|
| 137 |
|
| 138 |
+
function ProjectsPopupContent({ grain, rowKey, title, onClose, onChanged }: ProjectsPopupProps) {
|
| 139 |
+
const [view, setView] = useState<PopupView>({ kind: "list" });
|
|
|
|
|
|
|
| 140 |
const [read, setRead] = useState<ProjectList | null>(null);
|
| 141 |
const [error, setError] = useState("");
|
| 142 |
const [tick, setTick] = useState(0);
|
|
|
|
| 156 |
const label = `${GRAIN_LABEL[grain]} ${rowKey}`;
|
| 157 |
|
| 158 |
const actions = view.kind === "detail"
|
| 159 |
+
? <button type="button" className="lp-proj-btn lp-proj-btn-quiet" onClick={() => setView({ kind: "list" })}>All projects</button>
|
| 160 |
: view.kind === "list" && canEdit
|
| 161 |
? <>
|
| 162 |
<button type="button" className="lp-proj-btn lp-proj-btn-quiet" onClick={() => setView({ kind: "add" })}>Add existing project</button>
|
|
|
|
| 183 |
: null}
|
| 184 |
{view.kind === "detail"
|
| 185 |
? <ProjectDetail key={view.id} id={view.id} canEdit={canEdit} rowScope={{ grain, key: rowKey }}
|
|
|
|
|
|
|
| 186 |
onChanged={changed} onDeleted={onClose} onCancel={() => setView({ kind: "list" })} />
|
| 187 |
: null}
|
| 188 |
{view.kind === "new" && canEdit
|
|
|
|
| 204 |
|
| 205 |
// ---- The Projects tab: every project, the same rows ----
|
| 206 |
|
| 207 |
+
export function ProjectsPage() {
|
| 208 |
const [read, setRead] = useState<ProjectList | null>(null);
|
| 209 |
const [error, setError] = useState("");
|
| 210 |
const [tick, setTick] = useState(0);
|
|
|
|
| 241 |
{open
|
| 242 |
? <CenteredPopup className="lp-proj-popup" title="Project" onClose={() => setOpen(null)}>
|
| 243 |
<ProjectDetail key={open} id={open} canEdit={Boolean(read?.canEdit)}
|
|
|
|
| 244 |
onChanged={reload} onDeleted={() => { setOpen(null); reload(); }} onCancel={() => setOpen(null)} />
|
| 245 |
</CenteredPopup>
|
| 246 |
: null}
|
web/src/apps/amazon-profit/ProjectsShared.tsx
CHANGED
|
@@ -1,4 +1,4 @@
|
|
| 1 |
-
import { createContext, useContext, useEffect, useId, useRef, useState, type
|
| 2 |
import { createPortal } from "react-dom";
|
| 3 |
import { CellSignalHigh } from "@phosphor-icons/react/dist/csr/CellSignalHigh";
|
| 4 |
import { CellSignalMedium } from "@phosphor-icons/react/dist/csr/CellSignalMedium";
|
|
@@ -35,7 +35,6 @@ const STATUS_GROUPS: Array<{ label: string; statuses: readonly Status[] }> = [
|
|
| 35 |
|
| 36 |
export interface Project {
|
| 37 |
id: string;
|
| 38 |
-
type?: "project" | "pdp";
|
| 39 |
name: string;
|
| 40 |
status: Status;
|
| 41 |
priority: Priority;
|
|
@@ -338,20 +337,6 @@ export function Owner({ name, compact = false }: { name: string | null; compact?
|
|
| 338 |
// ---- The shared popup shell (S82-T90 reuses it for Competitors) ----
|
| 339 |
|
| 340 |
const ProjectFooterHost = createContext<HTMLElement | null>(null);
|
| 341 |
-
export function projectMenuKeys(event: KeyboardEvent<HTMLElement>) {
|
| 342 |
-
const items = Array.from(event.currentTarget.querySelectorAll<HTMLElement>('[role="menuitem"]:not(:disabled)'));
|
| 343 |
-
const index = items.indexOf(document.activeElement as HTMLElement);
|
| 344 |
-
const next = event.key === "ArrowDown" ? items[(index + 1) % items.length]
|
| 345 |
-
: event.key === "ArrowUp" ? items[(index - 1 + items.length) % items.length]
|
| 346 |
-
: event.key === "Home" ? items[0] : event.key === "End" ? items.at(-1) : undefined;
|
| 347 |
-
if (next) { event.preventDefault(); next.focus(); }
|
| 348 |
-
}
|
| 349 |
-
|
| 350 |
-
const ProjectHeaderHost = createContext<HTMLElement | null>(null);
|
| 351 |
-
export function ProjectHeader({ children }: { children: ReactNode }) {
|
| 352 |
-
const host = useContext(ProjectHeaderHost);
|
| 353 |
-
return host ? createPortal(children, host) : null;
|
| 354 |
-
}
|
| 355 |
export function ProjectFooter({ children }: { children: ReactNode }) {
|
| 356 |
const host = useContext(ProjectFooterHost);
|
| 357 |
return host ? createPortal(children, host) : <>{children}</>;
|
|
@@ -372,7 +357,6 @@ export interface CenteredPopupProps {
|
|
| 372 |
export function CenteredPopup({ title, subtitle, actions, onClose, children, className, size = "wide" }: CenteredPopupProps) {
|
| 373 |
const panelRef = useRef<HTMLDivElement>(null);
|
| 374 |
const [footer, setFooter] = useState<HTMLDivElement | null>(null);
|
| 375 |
-
const [header, setHeader] = useState<HTMLSpanElement | null>(null);
|
| 376 |
const titleId = useId();
|
| 377 |
useOverlayLayer({ panelRef, onDismiss: onClose, trapFocus: true, restoreFocus: true, initialFocus: "first", dismissOnOutside: false });
|
| 378 |
return <BodyPortal>
|
|
@@ -388,14 +372,13 @@ export function CenteredPopup({ title, subtitle, actions, onClose, children, cla
|
|
| 388 |
</div>
|
| 389 |
<div className="lp-cpop-actions">
|
| 390 |
{actions}
|
| 391 |
-
{className?.includes("lp-proj-popup") ? <span className="pdp-project-header" ref={setHeader} /> : null}
|
| 392 |
<button type="button" className="lp-cpop-close" aria-label="Close" onClick={onClose}><X size={14} aria-hidden="true" /></button>
|
| 393 |
</div>
|
| 394 |
</header>
|
| 395 |
-
<
|
| 396 |
<div className="lp-cpop-body">{children}</div>
|
| 397 |
{className?.includes("lp-proj-popup") ? <div className="lp-proj-popup-footer" ref={setFooter} /> : null}
|
| 398 |
-
</ProjectFooterHost.Provider>
|
| 399 |
</div>
|
| 400 |
</div>
|
| 401 |
</BodyPortal>;
|
|
|
|
| 1 |
+
import { createContext, useContext, useEffect, useId, useRef, useState, type ReactNode } from "react";
|
| 2 |
import { createPortal } from "react-dom";
|
| 3 |
import { CellSignalHigh } from "@phosphor-icons/react/dist/csr/CellSignalHigh";
|
| 4 |
import { CellSignalMedium } from "@phosphor-icons/react/dist/csr/CellSignalMedium";
|
|
|
|
| 35 |
|
| 36 |
export interface Project {
|
| 37 |
id: string;
|
|
|
|
| 38 |
name: string;
|
| 39 |
status: Status;
|
| 40 |
priority: Priority;
|
|
|
|
| 337 |
// ---- The shared popup shell (S82-T90 reuses it for Competitors) ----
|
| 338 |
|
| 339 |
const ProjectFooterHost = createContext<HTMLElement | null>(null);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 340 |
export function ProjectFooter({ children }: { children: ReactNode }) {
|
| 341 |
const host = useContext(ProjectFooterHost);
|
| 342 |
return host ? createPortal(children, host) : <>{children}</>;
|
|
|
|
| 357 |
export function CenteredPopup({ title, subtitle, actions, onClose, children, className, size = "wide" }: CenteredPopupProps) {
|
| 358 |
const panelRef = useRef<HTMLDivElement>(null);
|
| 359 |
const [footer, setFooter] = useState<HTMLDivElement | null>(null);
|
|
|
|
| 360 |
const titleId = useId();
|
| 361 |
useOverlayLayer({ panelRef, onDismiss: onClose, trapFocus: true, restoreFocus: true, initialFocus: "first", dismissOnOutside: false });
|
| 362 |
return <BodyPortal>
|
|
|
|
| 372 |
</div>
|
| 373 |
<div className="lp-cpop-actions">
|
| 374 |
{actions}
|
|
|
|
| 375 |
<button type="button" className="lp-cpop-close" aria-label="Close" onClick={onClose}><X size={14} aria-hidden="true" /></button>
|
| 376 |
</div>
|
| 377 |
</header>
|
| 378 |
+
<ProjectFooterHost.Provider value={className?.includes("lp-proj-popup") ? footer : null}>
|
| 379 |
<div className="lp-cpop-body">{children}</div>
|
| 380 |
{className?.includes("lp-proj-popup") ? <div className="lp-proj-popup-footer" ref={setFooter} /> : null}
|
| 381 |
+
</ProjectFooterHost.Provider>
|
| 382 |
</div>
|
| 383 |
</div>
|
| 384 |
</BodyPortal>;
|
web/src/apps/templateMarks.tsx
CHANGED
|
@@ -85,10 +85,6 @@ export const TEMPLATE_MARKS: Readonly<Record<string, ReactElement>> = {
|
|
| 85 |
alerts: <>
|
| 86 |
<path className={INK} d="M5 17h14l-2-3V9a5 5 0 0 0-10 0v5zM10 20h4" />
|
| 87 |
</>,
|
| 88 |
-
reports: <>
|
| 89 |
-
<path className={ACCENT} d="M14 3v5h5z" />
|
| 90 |
-
<path className={INK} d="M14 3H5v18h14V8zM14 3v5h5M8 12h8M8 16h8" />
|
| 91 |
-
</>,
|
| 92 |
};
|
| 93 |
|
| 94 |
const AMAZON_MARKS = new Set([
|
|
|
|
| 85 |
alerts: <>
|
| 86 |
<path className={INK} d="M5 17h14l-2-3V9a5 5 0 0 0-10 0v5zM10 20h4" />
|
| 87 |
</>,
|
|
|
|
|
|
|
|
|
|
|
|
|
| 88 |
};
|
| 89 |
|
| 90 |
const AMAZON_MARKS = new Set([
|
web/src/automation/AutomationDetail.tsx
CHANGED
|
@@ -22,7 +22,7 @@ import PanelToggleIcon from "../shell/PanelToggleIcon";
|
|
| 22 |
// what this codebase treats as a defect rather than a summary.
|
| 23 |
// ---------------------------------------------------------------------------
|
| 24 |
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
|
| 25 |
-
import {
|
| 26 |
|
| 27 |
import AutomationBuilder, { SavedQuerySchedules } from "./AutomationBuilder";
|
| 28 |
import HarnessPanel from "./HarnessPanel";
|
|
@@ -335,6 +335,28 @@ function RetiredAutomationResult({ automation }: { automation: Automation }) {
|
|
| 335 |
);
|
| 336 |
}
|
| 337 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 338 |
export default function AutomationDetailView(props: Props) {
|
| 339 |
return props.automation.retired
|
| 340 |
? <RetiredAutomationResult automation={props.automation} />
|
|
|
|
| 22 |
// what this codebase treats as a defect rather than a summary.
|
| 23 |
// ---------------------------------------------------------------------------
|
| 24 |
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
|
| 25 |
+
import { TOAST_EVENT } from "../apiContract";
|
| 26 |
|
| 27 |
import AutomationBuilder, { SavedQuerySchedules } from "./AutomationBuilder";
|
| 28 |
import HarnessPanel from "./HarnessPanel";
|
|
|
|
| 335 |
);
|
| 336 |
}
|
| 337 |
|
| 338 |
+
function useRefusalToast() {
|
| 339 |
+
const [messages, setMessages] = useState<{ id: number; text: string }[]>([]);
|
| 340 |
+
const sequence = useRef(0);
|
| 341 |
+
const delivered = useRef(0);
|
| 342 |
+
useEffect(() => {
|
| 343 |
+
if (!messages.length) return;
|
| 344 |
+
const through = messages[messages.length - 1].id;
|
| 345 |
+
for (const message of messages) {
|
| 346 |
+
if (message.id <= delivered.current) continue;
|
| 347 |
+
delivered.current = message.id;
|
| 348 |
+
const text = message.text;
|
| 349 |
+
window.dispatchEvent(new CustomEvent(TOAST_EVENT, { detail: { text, tone: "error" } }));
|
| 350 |
+
}
|
| 351 |
+
setMessages((current) => current.filter((message) => message.id > through));
|
| 352 |
+
}, [messages]);
|
| 353 |
+
return useCallback((text: string) => {
|
| 354 |
+
if (!text) return;
|
| 355 |
+
const message = { id: ++sequence.current, text };
|
| 356 |
+
setMessages((current) => [...current, message]);
|
| 357 |
+
}, []);
|
| 358 |
+
}
|
| 359 |
+
|
| 360 |
export default function AutomationDetailView(props: Props) {
|
| 361 |
return props.automation.retired
|
| 362 |
? <RetiredAutomationResult automation={props.automation} />
|
web/src/automation/AutomationSurface.tsx
CHANGED
|
@@ -18,8 +18,7 @@ import type { KeyboardEvent as ReactKeyboardEvent } from "react";
|
|
| 18 |
import { Suspense, lazy, useCallback, useEffect, useRef, useState } from "react";
|
| 19 |
|
| 20 |
import type { AutomationOpenDetail } from "../apiContract";
|
| 21 |
-
import { AUTOMATION_OPEN_EVENT, UNAUTHORIZED_EVENT, currentTenant } from "../apiContract";
|
| 22 |
-
import { useRefusalToast } from "../shell/ToastStack";
|
| 23 |
// ⭐ WAVE 26 · ITEM 18 / R14 + C7 — THE LOOPABLE LOOP MARK, CONSUMED, NEVER REDRAWN.
|
| 24 |
// The owner's reason is the mark's own geometry: *"these automations are basically loops."*
|
| 25 |
// C7 says consume the existing icon component and do not inline a new SVG path, and
|
|
@@ -212,6 +211,27 @@ if (typeof window !== "undefined") {
|
|
| 212 |
* that a database can be created BY asking for an automation.
|
| 213 |
*/
|
| 214 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 215 |
export default function AutomationSurface() {
|
| 216 |
/* ⭐ T22 — SEEDED FROM THE CLIENT MEMO, so a revisit inside the freshness window never passes
|
| 217 |
through `data === null` and therefore never paints the skeleton. The lazy initialiser runs
|
|
|
|
| 18 |
import { Suspense, lazy, useCallback, useEffect, useRef, useState } from "react";
|
| 19 |
|
| 20 |
import type { AutomationOpenDetail } from "../apiContract";
|
| 21 |
+
import { AUTOMATION_OPEN_EVENT, TOAST_EVENT, UNAUTHORIZED_EVENT, currentTenant } from "../apiContract";
|
|
|
|
| 22 |
// ⭐ WAVE 26 · ITEM 18 / R14 + C7 — THE LOOPABLE LOOP MARK, CONSUMED, NEVER REDRAWN.
|
| 23 |
// The owner's reason is the mark's own geometry: *"these automations are basically loops."*
|
| 24 |
// C7 says consume the existing icon component and do not inline a new SVG path, and
|
|
|
|
| 211 |
* that a database can be created BY asking for an automation.
|
| 212 |
*/
|
| 213 |
|
| 214 |
+
function useRefusalToast() {
|
| 215 |
+
const [messages, setMessages] = useState<{ id: number; text: string }[]>([]);
|
| 216 |
+
const sequence = useRef(0);
|
| 217 |
+
const delivered = useRef(0);
|
| 218 |
+
useEffect(() => {
|
| 219 |
+
if (!messages.length) return;
|
| 220 |
+
const through = messages[messages.length - 1].id;
|
| 221 |
+
for (const { id, text } of messages) {
|
| 222 |
+
if (id <= delivered.current) continue;
|
| 223 |
+
delivered.current = id;
|
| 224 |
+
window.dispatchEvent(new CustomEvent(TOAST_EVENT, { detail: { text, tone: "error" } }));
|
| 225 |
+
}
|
| 226 |
+
setMessages((current) => current.filter((message) => message.id > through));
|
| 227 |
+
}, [messages]);
|
| 228 |
+
return (text: string) => {
|
| 229 |
+
if (!text) return;
|
| 230 |
+
const message = { id: ++sequence.current, text };
|
| 231 |
+
setMessages((current) => [...current, message]);
|
| 232 |
+
};
|
| 233 |
+
}
|
| 234 |
+
|
| 235 |
export default function AutomationSurface() {
|
| 236 |
/* ⭐ T22 — SEEDED FROM THE CLIENT MEMO, so a revisit inside the freshness window never passes
|
| 237 |
through `data === null` and therefore never paints the skeleton. The lazy initialiser runs
|