diff --git a/RELEASES.json b/RELEASES.json index 9384bd67bb0fd23f0dd849538e285a71c8b20c08..0139a412466f9ba1bdd97505c566951e9d0f620d 100644 --- a/RELEASES.json +++ b/RELEASES.json @@ -1,4 +1,4 @@ { - "current": "2532efa50c", + "current": "b1a532ae668", "releases": [] } \ No newline at end of file diff --git a/VERSION b/VERSION index 73fda7a816ecfc74e5bf0054b5b5c9614d473204..b06d7249487ff9c92fb0a9a544fd629e7e8766d6 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2532efa50c +b1a532ae668 diff --git a/api/amazon_quotes_loop.py b/api/amazon_quotes_loop.py index bc940e798b934cdd34296beeff9146ad21e7b0c9..631e8d3a1a28d0b83356308f7111ec5343e5eefa 100644 --- a/api/amazon_quotes_loop.py +++ b/api/amazon_quotes_loop.py @@ -9,12 +9,10 @@ import logging from pathlib import Path import threading import time -import uuid JOB = "amazon-rival-quotes" #: The loop asks each tenant whether its daily run is due this often. POLL_SECONDS = 600 -SNAPSHOT_TABLES = ("competitor_snapshots", "competitor_images") #: Called as `fn(tenant, changed_asins)` after a run finishes, where `changed_asins` is a tuple of the #: ASINs whose newest quote changed. Each import is lazy and a failure is one log line, never a failed run. @@ -28,29 +26,17 @@ _thread = None _stop = None -class SnapshotBlocked(RuntimeError): - """A safe admission cause that can be recorded without exposing provider details.""" - - def _say(message): print(f"[{JOB}] {message}", flush=True) def readers(fields): - """Read prices and complete facts with the tenant credential and the shared Amazon pacers.""" + """The three SP-API reads of one tenant's credential. Each takes a list of at most 20 ASINs.""" from harness import amazon_client as cx - from harness import amazon_competitor_facts as facts - - def complete(asins): - catalog, offers = facts.read_facts(fields, asins), facts.read_offers(fields, asins) - return {asin: {**catalog[asin], **{key: offers[asin][key] for key in ("prime", "offers")}, - "cause": " ".join(c for c in (catalog[asin]["cause"], offers[asin]["cause"]) if c) or None} - for asin in asins} - return {"offers": lambda asins: cx.item_offers(fields, asins), # The pricing sync draws this same `competitive_summary` pacer in this worker process. "summaries": lambda asins: cx.competitive_summaries(fields, asins), - "facts": complete} + "ranks": lambda asins: cx.catalog_sales_ranks(fields, asins)} @contextmanager @@ -78,69 +64,9 @@ def _after_quotes(tenant, changed): "the run result is unchanged.") -def _snapshot_cause(runtime, *, check_lease=True): - from harness import amazon_store - - enabled = amazon_store.read_database_enablement(runtime) - if not enabled["available"]: - return "Competitor database settings could not be read. Restore settings access and retry the run." - if not all(enabled["databases"].get("ut_amzn_" + table) for table in SNAPSHOT_TABLES): - return "A competitor database is disabled. Enable competitor snapshots and images in Settings, Connectors and retry." - if check_lease and amazon_store.first_load_job_live(runtime): - return "The Amazon mirror writer lease is held. Let the current writer finish and retry the run." - return None - - -def _publish_snapshots(tenant, runtime, result): - from harness import amazon_competitor_facts as facts, amazon_store, mirror_store - - lock = mirror_store.for_connector("amazon").lock - if not lock.acquire(blocking=False): - raise SnapshotBlocked("The Amazon mirror writer is busy.") - holder = f"{JOB}:{result['id']}" - leased = False - try: - leased = amazon_store.acquire_amazon_lease(runtime, holder) - if not leased: - raise SnapshotBlocked("The Amazon mirror writer lease is held.") - cause = _snapshot_cause(runtime, check_lease=False) - if cause: - raise SnapshotBlocked(cause) - report = {"tables": {}, "problems": []} - with mirror_store.sync_mirror("amazon", tenant, runtime, report=report, expected_tables=SNAPSHOT_TABLES): - con, _path = amazon_store._connect_tenant_mirror(tenant, runtime) - try: - stored = facts.import_snapshots(con, result["snapshots"]) - for table in SNAPSHOT_TABLES: - total = con.execute(f"SELECT count(*) FROM amzn_{table}").fetchone()[0] - report["tables"][table] = {"state": "done", "rows": total} - finally: - con.close() - if report["problems"]: - raise RuntimeError("Competitor mirror publication failed") - result["counts"].update(snapshots=stored["snapshots"], images=stored["images"]) - finally: - try: - if leased: - amazon_store.release_amazon_lease(runtime, holder) - finally: - lock.release() - - -def _run_blocked(runtime, cause): - from core import amazon_rivals as rivals - - run_id = uuid.uuid4().hex - rivals.start_run(runtime, run_id, asin_total=0) - return {"id": run_id, "progress": 0, "cause": cause, - "counts": {"facts": dict.fromkeys(rivals.FACT_COUNT_COLUMNS, 0)}, "changed": (), "snapshots": []} - - def run_tenant(tenant, runtime, *, now=None, fields=None, sleep=time.sleep): """Run today's quotes for one tenant when due. None when it is not due or has no Amazon store.""" from core import amazon_rivals as rivals - if str(tenant) != str(runtime.key) or str(runtime.key) != str(runtime.tenant.key): - raise ValueError("Competitor quotes require the matching tenant runtime") try: if not rivals.due(runtime, now=now): return None @@ -154,23 +80,9 @@ def run_tenant(tenant, runtime, *, now=None, fields=None, sleep=time.sleep): fields = amazon_store.credential(tenant, runtime) if not fields: return None - started = time.monotonic() - cause = _snapshot_cause(runtime) - result = (_run_blocked(runtime, cause) if cause else - rivals.run_quotes(runtime, lambda: _mirror(runtime), **readers(fields), - seller_id=str(fields.get("seller_id") or "").strip() or None, now=now, - log=lambda message: _say(f"tenant={tenant} {message}"), sleep=sleep)) - if result["snapshots"]: - try: - _publish_snapshots(tenant, runtime, result) - except Exception as exc: - cause = (str(exc) if isinstance(exc, SnapshotBlocked) else - f"Competitor snapshots could not be published ({type(exc).__name__}).") - result["cause"] = (f"{cause} Check database settings and mirror access, let any writer finish, " - f"and retry within {rivals.RETRY_RUN_HOURS} hours.") - result["counts"]["elapsed_seconds"] = round(time.monotonic() - started, 3) - rivals.mark_run(runtime, result["id"], progress=result["progress"], finished=True, - counts=result["counts"], cause=result["cause"]) + result = rivals.run_quotes(runtime, lambda: _mirror(runtime), **readers(fields), + seller_id=str(fields.get("seller_id") or "").strip() or None, now=now, + log=lambda message: _say(f"tenant={tenant} {message}"), sleep=sleep) if result["cause"] is None: _after_quotes(tenant, result["changed"]) return result diff --git a/api/amazon_relational.py b/api/amazon_relational.py index 53d40081bd1c3945907d39b4822779f86a1f3b96..5dfa1369ea500ef3a22bbdf97259d115c9a409b2 100644 --- a/api/amazon_relational.py +++ b/api/amazon_relational.py @@ -650,8 +650,6 @@ def _fields_for(cur, key, *, read_through=False): "pinned": name == pinned, **({"description": amazon_schema.REVIEW_UNAVAILABLE_NOTE} if name == "asin" and key in (PREFIX + "review_topics", PREFIX + "review_trends") else {}), - **({"description": amazon_schema.SQP_LIMIT_NOTE} - if name == "asin" and key == PREFIX + "search_query_performance" else {}), **({"format": fmt} if fmt else {}), }, flow="amazon_relational")) for lookup in derived.get(name, ()): @@ -725,8 +723,7 @@ READ_THROUGH = frozenset( and (spec.get("report") or spec.get("api") or spec.get("monthly")))) | frozenset( PREFIX + table for table in amazon_store.DERIVED_TABLES) | frozenset( # Change history grows without a cap (standing rule 1), so it is never stored. - PREFIX + table for table in amazon_store.HISTORY_TABLES) | frozenset(( - PREFIX + "competitor_snapshots", PREFIX + "competitor_images")) + PREFIX + table for table in amazon_store.HISTORY_TABLES) def _quoted(name): @@ -1601,8 +1598,6 @@ def apply_plan(rt, built, username="amazon", today=None): doc[key]["description"] = ads_descriptions[key] if key in (PREFIX + "review_topics", PREFIX + "review_trends"): doc[key]["description"] = amazon_schema.REVIEW_UNAVAILABLE_NOTE - if key == PREFIX + "search_query_performance": - doc[key]["description"] = amazon_schema.SQP_LIMIT_NOTE if key == PREFIX + "listing_items": doc[key]["amazonListingIdentity"] = listing_version return doc diff --git a/api/app_templates.py b/api/app_templates.py index 09211ad2b0ea6323891e476567e8d8f62b617d2e..53f494823e188440830ec798e8dfc0afa49837f2 100644 --- a/api/app_templates.py +++ b/api/app_templates.py @@ -127,22 +127,12 @@ TEMPLATES = ( # Project management reads its own module tables (ut_amazon_*), so it declares no # source: a cold mirror must not refuse an App whose rows live in the store. _amazon("amazon_profit", "Project management", "See profit, ads and stock for every SKU.", - "Create value", (), - optional_sources=("ut_amzn_search_query_performance", "ut_amzn_order_items", - "ut_amzn_brand_analytics", "ut_amzn_listings", "ut_amzn_catalog_items", - "ut_amzn_listing_items", "ut_amzn_competitor_snapshots", - "ut_amzn_competitor_images"), - optional_connectors=("odoo",), proposes=False), + "Create value", (), optional_connectors=("odoo",), proposes=False), {"kind": "alerts", "label": "Alerts", "purpose": "Act on changes in your databases.", "connector": None, "connectors": (), "permission": "automation", "tags": ["Automation"], "sources": (), "optional_sources": (), "optional_connectors": (), "process": "None", "mark": "alerts", "module": None, "router": "routes_automation", "view": "alerts", "tools": (), "proposes": False}, - {"kind": "reports", "label": "Reports", "purpose": "Download reports shared with you.", - "connector": "odoo", "connectors": ("odoo",), "tags": ["Odoo"], "sources": (), - "optional_sources": (), "optional_connectors": (), "process": "None", "mark": "reports", - "module": "modules.reports", "router": "routes_reports", "view": "reports", - "tools": (), "proposes": False}, ) if not all(validate_connectors(row) for row in TEMPLATES): raise ValueError("An App template omits a source connector.") diff --git a/api/application_registry.py b/api/application_registry.py index 4f177256f6a0a5c66396d9c29a3a63a7b7176900..bc5c7f81a18f77ee9b642ce384013d5e649c21a9 100644 --- a/api/application_registry.py +++ b/api/application_registry.py @@ -624,16 +624,6 @@ def _source_rows_exist(session, sources): return False -def _reports_readable(runtime, user, is_admin): - """Report grants admit the Reports App independently of its App sharing record.""" - try: - from modules import reports - except Exception as exc: # A failed Reports import withholds its App. - raise_if_resource_error(exc) - return False - return bool(is_admin or reports.granted_keys(runtime, user)) - - def _readable_template_app(session, app_id): """Resolve App identity before reading any source or exposing a refusal.""" from deps import err @@ -645,10 +635,7 @@ def _readable_template_app(session, app_id): except Exception: raise err(503, "app_unavailable", "This App cannot be read right now.") from None if (not isinstance(app, dict) or app.get("id") != app_id or app.get("tenant") != session.tenant - or app.get("retired") is True - or not (_reports_readable(session.runtime, session.uname, session.admin) - if app.get("templateKind") == "reports" - else may_read(app, session.uname, session.admin))): + or app.get("retired") is True or not may_read(app, session.uname, session.admin)): raise err(404, "app_not_found", "This App is unavailable or is not shared with you.") kind = app.get("templateKind") catalogue = _app_templates.BY_KIND.get(str(kind or "")) @@ -1318,7 +1305,7 @@ def validate_application(payload, tenant, *, semantic_records=None, field_contra def _public(app, user, is_admin=False, *, allowed_databases=None, target_database=None, - is_default=False, readable=None): + is_default=False): row=copy.deepcopy(_app_projection(app)) row.setdefault("origin", None) if row.get("type") == "code": @@ -1327,10 +1314,9 @@ def _public(app, user, is_admin=False, *, allowed_databases=None, target_databas compatible = [database for database in row.get("compatibleDatabases") or () if database in _allowed(row.get("compatibleDatabases"), allowed_databases)] row["compatibleDatabases"] = compatible - app_readable = may_read(app, user, is_admin) - row["capabilities"]={"read":app_readable if readable is None else readable,"edit":may_edit(app,user,is_admin), + row["capabilities"]={"read":may_read(app,user,is_admin),"edit":may_edit(app,user,is_admin), "share":may_edit(app,user,is_admin),"delete":may_edit(app,user,is_admin), - "duplicate":app_readable,"star":app_readable, + "duplicate":may_read(app,user,is_admin),"star":may_read(app,user,is_admin), "default":bool(is_admin)} row["starred"] = user in (app.get("starredBy") or ()) row["isDefault"] = bool(is_default) @@ -1391,7 +1377,6 @@ def list_apps(runtime, user, is_admin=False, *, database=None, kind=None, starre """List Applications. Unfiltered is the App home; a database filter is how a View resolves.""" doc = load(runtime) rows=[] - reports_reader = None for app in doc["applications"].values(): if (isinstance(app, dict) and getattr(runtime, "key", None) and app.get("tenant") != runtime.key): @@ -1401,19 +1386,13 @@ def list_apps(runtime, user, is_admin=False, *, database=None, kind=None, starre if (isinstance(app, dict) and app.get("templateKind") == "alerts" and not (may_open or _user_wall(runtime, user))("automation")): continue - report_app = isinstance(app, dict) and app.get("templateKind") == "reports" - if report_app: - if reports_reader is None: - reports_reader = _reports_readable(runtime, user, is_admin) - if not reports_reader or app.get("retired") is True: - continue if isinstance(app, dict) and app.get("templateKind") in ("customer_card", FINANCE_TEMPLATE_KIND): continue if database: # A View resolves its App only through this read, so a retired seed stays listed here. - if not report_app and not may_read(app,user,is_admin): continue + if not may_read(app,user,is_admin): continue if database not in (app.get("compatibleDatabases") or ()): continue - elif not report_app and not _listed(app, user): continue + elif not _listed(app, user): continue if (not _database_free(app) and not _allowed(app.get("compatibleDatabases"), allowed_databases)): catalogue = _app_templates.BY_KIND.get(str(app.get("templateKind") or "")) @@ -1425,7 +1404,6 @@ def list_apps(runtime, user, is_admin=False, *, database=None, kind=None, starre default_key = f"{database}::{app.get('type')}" if database else "" rows.append(_public(app,user,is_admin,allowed_databases=allowed_databases, target_database=database, - readable=True if report_app else None, is_default=bool(default_key and doc["defaults"].get(default_key)==app.get("id")))) # S67-T39: the dependent-View count rides a DATABASE-filtered read only, and only for rows # the caller may delete, so the unfiltered nav read (App home tiles) pays nothing for it. diff --git a/api/connectors_amazon.py b/api/connectors_amazon.py index 5357479870133c043fdc7dca3acb206884e669ce..db1a5915be16a299703bfa2f4158270fb67aea72 100644 --- a/api/connectors_amazon.py +++ b/api/connectors_amazon.py @@ -611,11 +611,16 @@ def sync_state(rt, tenant_key, *, cadence_seconds=None, now=None): answer[table]["warning"] = warning if table == az.SQP_TABLE or table in az.GROUP_A_TABLES or table in ("transactions", "brand_analytics"): answer[table]["limit"] = limit - for table in (*az.DERIVED_TABLES, "competitor_snapshots", "competitor_images"): - answer[table] = { - "rowsLoaded": mirrored.get(table), "lastSuccess": None, "inProgress": False, - "unknownCause": az.amazon_schema.PROVENANCE_COPY[az.amazon_schema.PROVENANCE[table]], - } + answer["sku_keywords"] = { + "rowsLoaded": mirrored.get("sku_keywords"), + "lastSuccess": None, + "inProgress": False, + "unknownCause": az.amazon_schema.PROVENANCE_COPY["derived"], + } + answer["keyword_ranks"] = { + "rowsLoaded": mirrored.get("keyword_ranks"), "lastSuccess": None, "inProgress": False, + "unknownCause": az.amazon_schema.PROVENANCE_COPY["apify_dataset"], + } # A history table is written in its source's transaction, so it shares the source's last success. for table in az.HISTORY_TABLES: rows = mirrored.get(table) diff --git a/api/external_freshness.py b/api/external_freshness.py index 7eb771cbeae127c1689c33fb3b9d26cec5eb53c7..09e06f19dc381e7504e81e57f5863fd582461af0 100644 --- a/api/external_freshness.py +++ b/api/external_freshness.py @@ -82,24 +82,18 @@ def _amazon_raw(table_key, source=None): return None -def amazon_sync_state(cur, rt=None, *, entry=None): +def amazon_sync_state(cur, rt=None): """`{entity: updated_at}` for every `done` Amazon `_sync_state` row, in one read.""" - if rt is not None or entry is not None: + if rt is not None: from core import connector_sync, connector_spec - if entry is None: - entry = connector_sync.status(rt, "amazon_sp", fresh=True) + entry = connector_sync.status(rt, "amazon_sp", fresh=True) if entry is not None: last_pass = entry.get("pass") or {} held = last_pass.get("held") or {} stamps = last_pass.get("tableSync") or {} tables = {row.source.removeprefix("amzn_") for row in connector_spec.AMAZON_TABLES} - state = {"amzn_sync_mark_" + name: stamp for name in tables - if (stamp := stamps.get(name) or (entry.get("lastGoodSyncAt") if name not in held else None))} - if cur is not None and any(name not in stamps for name in held): - previous = amazon_sync_state(cur) - state.update({"amzn_sync_mark_" + name: stamp for name in held if name not in stamps - and (stamp := _amazon_table_stamp(previous, name)) is not None}) - return state + return {"amzn_sync_mark_" + name: stamp for name in tables + if (stamp := stamps.get(name) or (entry.get("lastGoodSyncAt") if name not in held else None))} return {str(entity): at for entity, at in cur.execute( "SELECT entity, updated_at FROM _sync_state " "WHERE phase = 'done' AND starts_with(entity, 'amzn_')").fetchall()} @@ -141,8 +135,8 @@ def amazon_sync_stamp(cur, table_key, state=None): return _amazon_table_stamp(state, table) -def _amazon_refreshes(session, keys, *, entry, served=False): - """Use the supplied canonical entry or served receipts for authorized Amazon databases.""" +def _amazon_refreshes(session, keys): + """Read only sync metadata for the authorized Amazon databases on this tenant's cursor.""" import assistant_mirror cur, state, refreshed = None, None, {} @@ -154,7 +148,7 @@ def _amazon_refreshes(session, keys, *, entry, served=False): cur = None continue if state is None: - state = amazon_sync_state(cur, entry=None if served else entry) + state = amazon_sync_state(cur, session.runtime) refreshed[key] = amazon_sync_stamp(cur, key, state=state) except Exception: continue @@ -244,7 +238,7 @@ def _mirror_refreshes(session, definitions, keys): return refreshed -def _served_behind(session, key, *, worker=None): +def _served_behind(session, key): """process_role's install lag for the connector behind one mirror database, or None.""" key = str(key or "") connector = ("amazon" if key.startswith("ut_amzn_") else @@ -253,82 +247,49 @@ def _served_behind(session, key, *, worker=None): return None try: import process_role - if worker is None: - worker = process_role.stored_worker(session.runtime) - return process_role.served_behind(getattr(session.runtime, "key", None) or session.tenant, connector, - worker=worker) + return process_role.served_behind(getattr(session.runtime, "key", None) or session.tenant, connector) except Exception: return None -def _retained_fact(served, state, reason): - """Keep the successful source receipt while naming why its rows have not advanced.""" - fact = {"source_synced_at": utc_stamp(served), "source_sync": state, "source_sync_reason": reason} - if (day := _sync_date(served)) is not None: - fact["source_synced_on"] = day - return fact - - -def _stale_fact(served, lag=None): +def _stale_fact(served): """A fact for a mirror whose newest published sync this deployment could not install.""" day = _sync_date(served) or (utc_stamp(served) or "")[:10] or None reason = (f"This deployment still serves the data synced on {day}. The newest sync could not be installed." if day else "This deployment serves an older copy of the data. The newest sync could not be installed.") - if lag and lag.get("cause"): - reason += " " + lag["cause"] - if lag and (retry := utc_stamp(lag.get("nextRetryAt"))): - reason += f" Next retry at {retry}." - return _retained_fact(served, "stale", reason) - - -def _amazon_holds(entry, keys): - """Map each authorized table to its own canonical hold or its held input tables.""" - held = (entry.get("pass") or {}).get("held") or {} - result = {} - for key in keys: - raw = _amazon_raw(key) - table = raw.removeprefix("amzn_") if raw else "" - dependencies = (table, *_AMAZON_DERIVED_INPUTS.get(table, ())) - rows = [held[name] for name in dependencies if isinstance(held.get(name), dict)] - if rows: - result[key] = rows - return result - - -def _held_fact(served, entries): - """Name the kept rows and each stored hold cause without changing their source date.""" - import dash_norm + return {"source_synced_at": None, "source_sync": "stale", "source_sync_reason": reason} - day = _sync_date(served) or (utc_stamp(served) or "")[:10] - reason = f"Held at the rows synced on {day}." if day else "Held at the previously synced rows." - causes = dict.fromkeys(dash_norm.plain(entry.get("cause") or "The last sync did not finish this table.") - for entry in entries) - reason += " " + " ".join(causes) - retries = sorted({stamp for entry in entries if (stamp := utc_stamp(entry.get("nextTryAt")))}) - if retries: - reason += f" Next retry at {retries[0]}." - return _retained_fact(served, "stale", reason) +def snapshot(session, databases): + """Index source receipts and edit times once for this authorized discovery call.""" + from core import connector_sync, user_tables -def _edit_receipts(rt, edited): - """Read the latest edit for all personal databases in one history aggregate.""" + wanted = set(databases) + rt = session.runtime + try: + definitions = rt.get_projection(user_tables.STORE_KEY, drop=("rows",)) + if not isinstance(definitions, dict): + raise ValueError("source definitions unavailable") + except Exception: + return {key: {"source_synced_at": None, "source_sync": "never_synced", + "source_sync_reason": "The source's connector status could not be verified."} + for key in wanted} + lent = user_tables.lend(rt, **{user_tables.STORE_KEY: definitions}) + connected = {key for key in wanted if key in definitions and user_tables.is_connected(key, st=lent)} + edited = {key for key in wanted if key in definitions and key not in connected and key != "ut_activity"} edits = {} if edited: + # D-11777: edits append only to the history rows, so one SQL aggregate gives each table's + # newest edit. A store with no history backend (TypeError) or an unreachable one keeps the + # definition's `updated` stamp below and never fails discovery. try: from core import history_rows latest = history_rows.backend(rt).latest(sorted(edited)) - except Exception: # Unavailable history keeps the definition's recorded edit time. + except Exception: latest = {} for key, at in latest.items() if isinstance(latest, dict) else (): if key in edited: edits[key] = _latest(edits.get(key), at) - return edits - - -def _connector_receipts(rt, connected, definitions): - """Read matching jobs without borrowing another connection's successful sync.""" - from core import connector_sync - receipts = {} if connected: try: @@ -344,84 +305,27 @@ def _connector_receipts(rt, connected, definitions): or declared.get("connectionId") and job.get("connectionId") != declared["connectionId"]): continue receipts[key] = job - return receipts - - -class _StatusSnapshot: - """Keep the request's fresh canonical status available to readers that request freshness again.""" - - def __init__(self, runtime, statuses): - self._runtime, self._statuses = runtime, statuses - - def get(self, name, fresh=False): - from core import connector_sync - - if name == connector_sync.STATUS_KEY: - return self._statuses - return self._runtime.get(name, fresh=True) if fresh else self._runtime.get(name) - - def __getattr__(self, name): - return getattr(self._runtime, name) - - -def _odoo_freshness(rt): - """Read Odoo's metadata without restoring or querying its mirror.""" - import odoo_relational - from core import user_tables - - docs = {} - for name in (odoo_relational.CONFIG_KEY, user_tables.ODOO_SYNC_STATUS_KEY): - try: - docs[name] = rt.get(name) or {} - except Exception: # A missing status must not turn a discovery read into an error. - docs[name] = {} - return odoo_relational.sync_freshness(user_tables.lend(rt, **docs)) - - -def _synced_fact(values): - """Use a successful source timestamp or preserve an explicitly date-only receipt.""" - stamp = next((stamp for value in values if (stamp := utc_stamp(value)) is not None), None) - if stamp is not None: - return {"source_synced_at": stamp, "source_sync": "synced"} - day = next((day for value in values if (day := _sync_date(value)) is not None), None) - if day is not None: - return {"source_synced_on": day, "source_sync": "synced_date_only", - "source_sync_reason": "The source reports only a sync date, without a time."} - return None - - -def snapshot(session, databases): - """Index source receipts and edit times once for this authorized discovery call.""" - from core import connector_sync, user_tables - - wanted, rt = set(databases), session.runtime - try: - definitions = rt.get_projection(user_tables.STORE_KEY, drop=("rows",)) - if not isinstance(definitions, dict): - raise ValueError("source definitions unavailable") - except Exception: - return {key: {"source_synced_at": None, "source_sync": "never_synced", - "source_sync_reason": "The source's connector status could not be verified."} - for key in wanted} - lent = user_tables.lend(rt, **{user_tables.STORE_KEY: definitions}) - connected = {key for key in wanted if key in definitions and user_tables.is_connected(key, st=lent)} - edited = {key for key in wanted if key in definitions and key not in connected and key != "ut_activity"} - edits, receipts = _edit_receipts(rt, edited), _connector_receipts(rt, connected, definitions) odoo_keys = {key for key in connected if key.startswith("ut_odoo_")} | (wanted & {"customer_data", "product_data"}) + odoo = {} + if odoo_keys: + import odoo_relational + + docs = {} + for name in (odoo_relational.CONFIG_KEY, user_tables.ODOO_SYNC_STATUS_KEY): + try: + docs[name] = rt.get(name) or {} + except Exception: + docs[name] = {} + odoo = odoo_relational.sync_freshness(user_tables.lend(rt, **docs)) amazon_keys = {key for key in connected if key.startswith("ut_amzn_")} - statuses = connector_sync.status_all(rt, fresh=True) if amazon_keys or odoo_keys else {} - odoo = _odoo_freshness(_StatusSnapshot(rt, statuses)) if odoo_keys else {} - worker = (statuses.get("sync_worker") or {}).get("default") or {} - amazon_entry = (statuses.get("amazon_sp") or {}).get("default") - behind = {group: _served_behind(session, next(iter(keys)), worker=worker) - for group, keys in (("odoo", odoo_keys), ("amazon", amazon_keys)) if keys} - amazon = _amazon_refreshes(session, amazon_keys, entry=amazon_entry, - served=behind.get("amazon") is not None) if amazon_keys else {} - held = _amazon_holds(amazon_entry or {}, amazon_keys) + amazon = _amazon_refreshes(session, amazon_keys) if amazon_keys else {} mirror_keys = {key for key in connected - amazon_keys if not user_tables.materialises(key, st=lent, defn=definitions[key])} mirror = _mirror_refreshes(session, definitions, mirror_keys) if mirror_keys else {} facts = {} + # One install-lag read per connector: a secondary web worker reads it from the primary's file. + behind = {group: _served_behind(session, next(iter(keys))) + for group, keys in (("odoo", odoo_keys), ("amazon", amazon_keys)) if keys} odoo_failed = {str(item.get("key")) for item in (odoo.get("failedDatabases") or ()) if isinstance(item, dict) and item.get("key")} for key in wanted: @@ -432,10 +336,7 @@ def snapshot(session, databases): elif key == "ut_activity": fact["source_sync"] = "not_applicable" elif behind.get("amazon" if key in amazon_keys else "odoo" if key in odoo_keys else None) is not None: - fact = _stale_fact(amazon.get(key) if key in amazon_keys else mirror.get(key), - behind["amazon" if key in amazon_keys else "odoo"]) - elif key in held: - fact = _held_fact(amazon.get(key), held[key]) + fact = _stale_fact(amazon.get(key) if key in amazon_keys else mirror.get(key)) else: health = definition.get(user_tables.CONNECTOR_HEALTH_FIELD) or {} job = receipts.get(key) or {} @@ -447,9 +348,13 @@ def snapshot(session, databases): if job.get("state") == "ready": successes.append(job.get("at")) values = [amazon.get(key)] if key.startswith("ut_amzn_") else [mirror.get(key), *successes] - synced = _synced_fact(values) - if synced is not None: - fact.update(synced) + stamp = next((stamp for value in values if (stamp := utc_stamp(value)) is not None), None) + synced_on = next((day for value in values if (day := _sync_date(value)) is not None), None) + if stamp is not None: + fact.update(source_synced_at=stamp, source_sync="synced") + elif synced_on is not None: + fact.update(source_synced_on=synced_on, source_sync="synced_date_only", + source_sync_reason="The source reports only a sync date, without a time.") elif (own_failed or job.get("state") == "failed" or key in odoo_keys and odoo.get("phase") == "error"): fact["source_sync_reason"] = "The connector's latest sync failed before a successful sync was recorded." diff --git a/api/main.py b/api/main.py index ac90d6b651748ec5d11ef566e51b81cd641998f9..789bf4108d8d80831074b4365f1aa9e7ddf1ee38 100644 --- a/api/main.py +++ b/api/main.py @@ -758,13 +758,6 @@ import app_routes _mount_router("routes_security") # W59-T43: lane F declares the router; keep before static mount. _mount_router("routes_amazon_profit") # S78-T03: the Command Center refresh and settings doors. _mount_router("routes_amazon_projects") -_mount_router("routes_amazon_pdp") -_mount_router("routes_pdp_text") -_mount_router("routes_pdp_images") -if _mount_router("routes_pdp_publish"): - # Amazon fetches published image URLs without a session, so the media router carries no auth. - from routes_pdp_publish import media_router as _pdp_media_router - app.include_router(_pdp_media_router) _mount_router("routes_competitors") _mount_router("routes_odoo_attachments") # S78-T14 (lane G, C5): delivery photo doors; before static mount. _mount_router("routes_sku_actions") @@ -1350,8 +1343,6 @@ def _pull_amazon(why, vendor=True): # D-13614: before the lease check, since the chain that holds the lease is the writer that publishes. _install_amazon_publication(tenant_key, rt, why) if _amazon.first_load_job_live(rt): - from harness import mirror_store - mirror_store.record_sync_attempt("amazon", rt) print("[aios-api] the first-load job holds the Amazon writer lease", flush=True) return False # Hotfix 2026-09-24: an unfinished first load goes through the Sync now desk, whose @@ -1387,7 +1378,6 @@ def _pull_amazon(why, vendor=True): except Exception as exc: # A writer that took the lease after the first-load check owns Amazon: skip, record no pass. if isinstance(exc, RuntimeError) and "holds the amazon writer lease" in str(exc): - mirror_store.record_sync_attempt("amazon", rt) print(f"[aios-api] amazon sync skipped ({why}): another job holds the Amazon writer lease", flush=True) return False @@ -1759,16 +1749,8 @@ def _amazon_sync_receipt(report, rt=None): def _sync_now_detail(provider, tenant, *, ok, detail="", cause=None): - """Name a classified failure or a completed publication's typed entity hold.""" + """Record one classified failure; leave successful receipt text untouched.""" if ok: - if provider == "odoo": - from harness import mirror_store - held = mirror_store.odoo_moving_entities(cause) - if held: - entity = sorted(held)[0] - others = f" {len(held) - 1} other entities were held." if len(held) > 1 else "" - return (f"Odoo {entity} changed during all 3 full reads. Its previous rows were kept." - + others + " Other completed entities published.") return detail classifier, sentence = _sync_now_cause(cause) if classifier == "nothing_due": @@ -1969,35 +1951,9 @@ def _run_sync_requests(outcome=None, *, during_odoo=False, hold_first_load=None, **({"after_entity": _after_entity} if _after_entity else {})) if not report and selected_entities != []: raise _SyncNowRefused("Odoo did not run a sync. Check whether the connection is paused.") - held = mirror_store.odoo_moving_entities(report) - if any(row.get("phase") != "live" and name not in held for name, row in report.items()): + if any(row.get("phase") != "live" for row in report.values()): raise _SyncNowRefused("Odoo backfill is incomplete. Run Sync now again to continue it.") - waiting = mirror_store.pending_sync_entities("odoo", tenant, rt) - if held and waiting == []: - import json - store = mirror_store.for_connector("odoo") - with store.lock: - published, _ = store._connect(tenant, rt) - candidate = None - try: - path = rt.datastore_path.with_suffix(".odoo.candidate.duckdb") - candidate = datastore.apply_limits(datastore.duckdb.connect(str(path), read_only=True)) - retained = {row[0] for row in candidate.execute("SELECT name FROM _candidate_held").fetchall()} - publication = json.loads(store._download(rt, mirror_store._PASS_KEY) or b"{}") - installed = datastore._publication(published, "odoo") - if (retained == set(held) and publication.get("state") == "published" - and publication.get("tenant") == tenant and publication.get("connector") == "odoo" - and publication.get("completed_at") and installed.get("state") == "complete" - and publication.get("snapshot_id") - and publication.get("snapshot_id") == installed.get("snapshot_id")): - waiting = retained - finally: - if candidate is not None: - candidate.close() - published.close() - if ((held and (set(waiting or ()) != set(held) - or not any(row.get("phase") == "live" for row in report.values()))) - or (not held and waiting is not None)): + if mirror_store.pending_sync_entities("odoo", tenant, rt) is not None: raise _SyncNowRefused("Odoo publication is held. The previous snapshot remains available.") problems = [] elif provider == "amazon_sp": @@ -2090,15 +2046,14 @@ def _run_sync_requests(outcome=None, *, during_odoo=False, hold_first_load=None, detail = _sync_now_detail(provider, tenant, ok=False, detail=amazon_detail, cause=amazon_failure or problems[0]) else: - ok, detail = True, _sync_now_detail(provider, tenant, ok=True, detail="Sync finished.", - cause=report if provider == "odoo" else None) + ok, detail = True, "Sync finished." if amazon_detail and not problems: detail += " " + amazon_detail except _SyncNowRefused as exc: detail = str(exc) except Exception as exc: # noqa: BLE001 detail = _sync_now_detail(provider, tenant, ok=False, cause=exc) - if progress and not amazon_detail and not (provider == "odoo" and ok and held): + if progress and not amazon_detail: import re suffix = re.search(r" \(cause code: [a-z_]+\)$", detail) cause_code = suffix[0] if suffix else "" @@ -3565,8 +3520,6 @@ def _amazon_desk_pass(tenant, *, project=True): job.join() state["job"] = None _project_desk(state["outcome"], tenant) - import sync_runtime - sync_runtime.relay_at_boundary() receipt = connector_sync.sync_request("amazon_sp", tenant) if (not receipt or receipt["state"] != "queued" or receipt.get("requestedBy") == "first load" or max(receipt.get("notBefore") or 0, @@ -3884,6 +3837,7 @@ def _store_resync_loop(): _resync_status = globals().get("_publish_resync_state") if _resync_status: _resync_status(_cad["tenant"], started=True) + _pass_started_epoch = _t.time() try: selected_entities = _configured_mirror_entities(_cad["tenant"]) sync_report, delete_report, _err = _resync_mirror_pass(_manual, passes, selected_entities) @@ -3914,6 +3868,26 @@ def _store_resync_loop(): "source", "ready", at=RESYNC_STATE["lastPassAt"], cadence_seconds=_every) except Exception: # noqa: BLE001 pass # The resident mirror state above still reports this pass. + elif not _manual: + # A held pass that published this pass's snapshot is a success. A refused publication writes nothing. + try: + from datetime import datetime as _held_dt + from harness import runtime as _held_runtime + import routes_connectors as _held_receipts + _held_rt = _held_runtime.get_runtime(_cad["tenant"]) + _held_pass = (connector_sync.status( + _held_rt, "odoo", RESYNC_STATE.get("connectionId") or "default", + fresh=True) or {}).get("pass") or {} + _published = (connector_sync._utc_stamp(_held_pass["publishedAt"]) + if _held_pass.get("publishedAt") else None) + if _published and (_held_dt.fromisoformat(_published).timestamp() + >= int(_pass_started_epoch)): + RESYNC_STATE["lastSuccessAt"] = RESYNC_STATE["lastPassAt"] + _held_receipts.record_odoo_stage( + _held_rt, RESYNC_STATE.get("connectionId") or "default", + "source", "ready", at=_published, cadence_seconds=_every) + except Exception: # noqa: BLE001 + pass # The pass row in the status document still reports this publication. else: RESYNC_STATE["consecutiveFailures"] = int(RESYNC_STATE["consecutiveFailures"]) + 1 # D-314: the TYPE and the persistent/transient reading, both in the line and in the diff --git a/api/odoo_relational.py b/api/odoo_relational.py index 41a07c779b6c3a10b9cd0175403efe6046300ae6..6242e98c51b6198c83189cc8c7fa5a8b7a8a58e1 100644 --- a/api/odoo_relational.py +++ b/api/odoo_relational.py @@ -6483,7 +6483,13 @@ def sync_freshness(rt, now=None): status = status if isinstance(status, dict) else {} from core import connector_sync reader = getattr(connector_sync, "status", None) - canonical = reader(rt, "odoo", fresh=True) if reader is not None and rt is not None else None + canonical = None + if reader is not None and rt is not None: + # The resync loop writes under the keychain default id. Read that slot first, else "default". + default_id = default_connection_id(rt) + canonical = reader(rt, "odoo", default_id, fresh=True) if default_id else None + if not isinstance(canonical, dict): + canonical = reader(rt, "odoo", fresh=True) canonical = canonical if isinstance(canonical, dict) else None pass_status = _sync_pass_freshness(canonical, cfg, now) phase = str(status.get("phase") or "unknown") diff --git a/api/process_role.py b/api/process_role.py index 91f636f690fbb2d90d7c0f7d170b95f0e7a04d69..06c98d7a035d27d82ae2c29f4bd2bbdc2f39fce7 100644 --- a/api/process_role.py +++ b/api/process_role.py @@ -56,20 +56,6 @@ READY_FILE = "READY.json" PRIMARY_LOCK = "web-primary.lock" PRIMARY_FILE = "web-primary.json" INSTALL_ATTEMPTS = 3 -INSTALL_RETRY_SECONDS = 3600 -_INSTALL_CAUSES = ( - (TimeoutError, "The install timed out."), - (ConnectionError, "The saved sync could not be downloaded."), - (PermissionError, "The install could not access the saved sync."), - (FileNotFoundError, "The saved sync is missing a required file."), - (OSError, "The install could not read or write its files."), - (MemoryError, "The install ran out of memory."), - (ValueError, "The saved sync failed validation."), - (TypeError, "The saved sync has an invalid structure."), - (LookupError, "The saved sync is missing required data."), - (RuntimeError, "The install stopped with an internal error."), - (Exception, "The install failed unexpectedly."), -) INCOMPLETE_GRACE_SECONDS = 3600 #: The C-status provider key the sync worker writes its heartbeat, catch-up and mirror ids under. WORKER_STATUS = "sync_worker" @@ -550,34 +536,37 @@ def pending_handoffs(root=None): def _failure_map(root=None): - """Return each published snapshot's own failed install receipt.""" + """{(tenant, connector): {snapshotId, attempts, type, at}} for each published snapshot that failed to install.""" if not is_primary(): stored = primary_state(root).get("installFailures") or {} return {tuple(key.split("/", 1)): row for key, row in stored.items() if "/" in key and isinstance(row, dict)} - return {key: dict(row) for key, snapshot in dict(_STORED["mirrors"]).items() - if (row := _INSTALL_FAILURES.get((key, snapshot)))} - - -def served_behind(tenant, connector, root=None, *, worker=None): + last, failures = dict(LAST_INSTALL_ERROR), {} + for key, snapshot in dict(_STORED["mirrors"]).items(): + attempts = _INSTALL_FAILURES.get((key, snapshot), 0) + if attempts: + # LAST_INSTALL_ERROR is one slot: its type and time belong only to the snapshot it names. + same = (last.get("tenant"), last.get("connector"), last.get("snapshotId")) == (*key, snapshot) + failures[key] = {"snapshotId": snapshot, "attempts": attempts, + "type": last.get("type") if same else None, "at": last.get("at") if same else None} + return failures + + +def served_behind(tenant, connector, root=None): """`{published, served, attempts, error}` when the store published a snapshot this container failed to install. None when nothing is published, the published snapshot is served, or no install of it failed yet. """ key = (str(tenant or "").strip().lower(), str(connector)) - worker = worker or {} - published = ((worker.get("mirrors") or {}).get(key[1]) or {}).get("snapshotId") or _known_mirrors(root).get(key) + published = _known_mirrors(root).get(key) if not published: return None served = _installed_map(root).get(key) - failure = _failure_map(root).get(key) or (worker.get("installs") or {}).get(_install_key(key, published)) or {} - if (failure.get("tenant"), failure.get("connector"), failure.get("snapshotId")) != (*key, published): - return None + failure = _failure_map(root).get(key) or {} attempts = int(failure.get("attempts") or 0) if failure.get("snapshotId") == published else 0 if served == published or attempts < 1: return None - return {"published": published, "served": served or failure.get("served"), "attempts": attempts, - "error": failure.get("type"), "cause": failure.get("cause"), "nextRetryAt": failure.get("nextRetryAt")} + return {"published": published, "served": served, "attempts": attempts, "error": failure.get("type")} # ----------------------------------------------------------------------------- several web workers @@ -785,44 +774,6 @@ def _remove_dir(path): shutil.rmtree(path, ignore_errors=True) -def _install_key(key, snapshot): - return "/".join((*key, snapshot)) - - -def _install_due(key, snapshot, stamp): - failure = _INSTALL_FAILURES.get((key, snapshot)) or {} - return failure.get("attempts", 0) < INSTALL_ATTEMPTS or stamp >= failure["nextRetryAt"] - - -def _record_install(key, snapshot, stamp, *, exc=None, rt=None, changed=True): - """Keep one scoped install receipt and persist its safe cause and retry deadline.""" - from core import connector_sync - - identity = (key, snapshot) - was_failed = identity in _INSTALL_FAILURES - row = {"tenant": key[0], "connector": key[1], "snapshotId": snapshot, "at": stamp, - "state": "failed" if exc is not None else "installed"} - if exc is not None: - kind, cause = next((kind, cause) for kind, cause in _INSTALL_CAUSES if isinstance(exc, kind)) - attempts = (_INSTALL_FAILURES.get(identity) or {}).get("attempts", 0) + 1 - row.update(attempts=attempts, type=kind.__name__, cause=cause, served=_INSTALLED.get(key), - nextRetryAt=stamp + (INSTALL_RETRY_SECONDS if attempts >= INSTALL_ATTEMPTS else POLL_SECONDS)) - _INSTALL_FAILURES[identity] = row - LAST_INSTALL_ERROR.clear() - LAST_INSTALL_ERROR.update(row) - else: - row["changed"] = changed is not False - _INSTALL_FAILURES.pop(identity, None) - if (LAST_INSTALL_ERROR.get("tenant"), LAST_INSTALL_ERROR.get("connector"), - LAST_INSTALL_ERROR.get("snapshotId")) == (*key, snapshot): - LAST_INSTALL_ERROR.clear() - if rt is not None and (exc is not None or was_failed): - written = connector_sync.update_status(rt, WORKER_STATUS, "installs", {_install_key(key, snapshot): row}) - if not written: - print(f"[process-role] install status not stored for {key[0]}/{key[1]}", flush=True) - return row - - def install_ready(root=None, installer=None, *, now=None): """Install the newest READY snapshot per tenant and connector, then delete older hand-offs. @@ -835,22 +786,25 @@ def install_ready(root=None, installer=None, *, now=None): for key, rows in ready_handoffs(root).items(): completed_at, newest_dir, ready = rows[-1] snapshot = ready.get("snapshotId") or newest_dir.name - if _INSTALLED.get(key) != snapshot and _install_due(key, snapshot, stamp): + if _INSTALLED.get(key) != snapshot and _INSTALL_FAILURES.get((key, snapshot), 0) < INSTALL_ATTEMPTS: try: changed = installer(key[1], key[0], ready) - except Exception as exc: # Retry failed installs without exposing their diagnostic text. - failure = _record_install(key, snapshot, stamp, exc=exc) + except Exception as exc: # noqa: BLE001 + _INSTALL_FAILURES[(key, snapshot)] = _INSTALL_FAILURES.get((key, snapshot), 0) + 1 + LAST_INSTALL_ERROR.update(tenant=key[0], connector=key[1], snapshotId=snapshot, at=stamp, + attempts=_INSTALL_FAILURES[(key, snapshot)], type=type(exc).__name__) print(f"[process-role] hand-off install {key[0]}/{key[1]} {snapshot} failed " - f"({failure['type']})", flush=True) + f"({type(exc).__name__}: {exc})", flush=True) continue _INSTALLED[key] = snapshot - _record_install(key, snapshot, stamp, changed=changed) if changed is False: # Already resident: nothing was rebuilt, so this is not an install. for _at, older, _ready in rows[:-1]: _remove_dir(older) continue LAST_INSTALL.update(tenant=key[0], connector=key[1], snapshotId=snapshot, at=stamp) + if (LAST_INSTALL_ERROR.get("tenant"), LAST_INSTALL_ERROR.get("connector")) == key: + LAST_INSTALL_ERROR.clear() installed.append(key) print(f"[process-role] installed {key[1]} snapshot {snapshot} for {key[0]}", flush=True) for _at, older, _ready in rows[:-1]: @@ -908,31 +862,29 @@ def install_stored(tenant, rt, mirrors, installer=None, *, now=None): installer = installer or _install_from_store stamp = time.time() if now is None else now installed = [] - stored = (stored_worker(rt).get("installs") or {}) if rt is not None else {} for connector, row in sorted((mirrors or {}).items()): snapshot = row.get("snapshotId") if isinstance(row, dict) else None if not snapshot: continue - key = (str(tenant).strip().lower(), str(connector)) + key = (str(tenant), str(connector)) _STORED["mirrors"][key] = snapshot - previous = stored.get(_install_key(key, snapshot)) or {} - if (previous.get("state") == "failed" and (key, snapshot) not in _INSTALL_FAILURES - and (previous.get("tenant"), previous.get("connector"), previous.get("snapshotId")) == (*key, snapshot)): - _INSTALL_FAILURES[(key, snapshot)] = previous - if _INSTALLED.get(key) == snapshot or not _install_due(key, snapshot, stamp): + if _INSTALLED.get(key) == snapshot or _INSTALL_FAILURES.get((key, snapshot), 0) >= INSTALL_ATTEMPTS: continue try: changed = installer(key[1], key[0], rt, snapshot) - except Exception as exc: # Preserve the serving copy and retry this snapshot on its own deadline. - failure = _record_install(key, snapshot, stamp, exc=exc, rt=rt) + except Exception as exc: # noqa: BLE001 + _INSTALL_FAILURES[(key, snapshot)] = _INSTALL_FAILURES.get((key, snapshot), 0) + 1 + LAST_INSTALL_ERROR.update(tenant=key[0], connector=key[1], snapshotId=snapshot, at=stamp, + attempts=_INSTALL_FAILURES[(key, snapshot)], type=type(exc).__name__) print(f"[process-role] store install {key[0]}/{key[1]} {snapshot} failed " - f"({failure['type']})", flush=True) + f"({type(exc).__name__}: {exc})", flush=True) continue _INSTALLED[key] = snapshot - _record_install(key, snapshot, stamp, rt=rt, changed=changed) if changed is False: continue LAST_INSTALL.update(tenant=key[0], connector=key[1], snapshotId=snapshot, at=stamp) + if (LAST_INSTALL_ERROR.get("tenant"), LAST_INSTALL_ERROR.get("connector")) == key: + LAST_INSTALL_ERROR.clear() installed.append(key) print(f"[process-role] installed {key[1]} snapshot {snapshot} for {key[0]} from the store", flush=True) return installed @@ -1283,23 +1235,6 @@ def start_web_services(): return True -def _install_diagnostics(stored): - """Read this tenant's current failed snapshot and last changed install across web processes.""" - tenant = writer_tenant() - rows = {**(stored.get("installs") or {}), **{ - _install_key(key, snapshot): row for (key, snapshot), row in dict(_INSTALL_FAILURES).items()}} - rows = sorted((row for row in rows.values() if isinstance(row, dict) and row.get("tenant") == tenant), - key=lambda row: row.get("at", 0), reverse=True) - mirrors = {key[1]: snapshot for key, snapshot in _known_mirrors().items() if key[0] == tenant} - mirrors.update({connector: row.get("snapshotId") for connector, row in (stored.get("mirrors") or {}).items()}) - failure = next((row for row in rows if row.get("state") == "failed" - and mirrors.get(row["connector"]) == row["snapshotId"] - and _INSTALLED.get((tenant, row["connector"])) != row["snapshotId"]), None) - installed = next((row for row in rows if row.get("state") == "installed" and row.get("changed")), None) - local = dict(LAST_INSTALL) if LAST_INSTALL.get("tenant") == tenant else None - return local or installed, failure - - def status(): """{role, worker: {pid, alive, heartbeatAgeS, restarts, lastInstall}} for the settings response. @@ -1324,13 +1259,12 @@ def status(): fleet[slug] = {"pid": None, "alive": bool(row.get("alive")), "restarts": row.get("jobs", 0), "lastExit": None, "jobId": row.get("jobId")} import sync_runtime - last_install, last_error = _install_diagnostics(stored) return {"role": ROLE, "roleSource": ROLE_SOURCE, "webWorkers": web_workers(), "primary": is_primary(), "syncRuntime": sync_runtime.runtime_name()[0], "tenant": writer_tenant(), "scope": worker_scope(), "tenantWorkers": fleet, "tenantWorkersOverCap": list(_FLEET_STATE["overCap"]), "syncWorkersMax": sync_worker_cap(), "worker": {"pid": pid, "alive": alive, "heartbeatAgeS": age, - "restarts": _WORKER["restarts"], "lastInstall": last_install, - "lastInstallError": last_error, + "restarts": _WORKER["restarts"], "lastInstall": dict(LAST_INSTALL) or None, + "lastInstallError": dict(LAST_INSTALL_ERROR) or None, "attestation": worker_provenance.visible(stored) if stored.get("heartbeat") == beat else None}, "writer": {"armed": _WRITER["armed"], "held": _WRITER_HELD.is_set(), "holder": _WRITER["holder"]}} diff --git a/api/routes_amazon_projects.py b/api/routes_amazon_projects.py index f9d5f5709efad1bbc4f33bf163bd85d2280616db..7e3fe423357e2cce5e6bfdb37e2bac79b9e70f34 100644 --- a/api/routes_amazon_projects.py +++ b/api/routes_amazon_projects.py @@ -38,7 +38,7 @@ FIELDS = {"name": "name", "status": "status", "priority": "priority", "owner": " "completedOn": "completed_on", "resultConfigs": "result_configs", "contributors": "contributors"} #: Project JSON keys a client may echo back. They are ignored, never written. READ_ONLY = frozenset(("id", "health", "targetCount", "createdAt", "updatedAt", "createdBy", - "rollup", "impact", "via", "figures", "type", "listed")) + "rollup", "impact", "via", "figures")) def _call(fn, *args, **kwargs): @@ -116,7 +116,6 @@ def _project(record, today): impact = dict(record.get("impact") or {}) impact.pop(projects.RESULT_CONFIGS_KEY, None) return {"id": record["id"], "name": record["name"], "status": record["status"], - "type": record.get("type", "project"), "listed": record.get("listed", True), "priority": record["priority"], "owner": identity["name"] if identity else record["owner"], "ownerUsername": identity["username"] if identity else None, "startOn": record["start_on"], "targetOn": record["target_on"], "progress": record["progress"], "notes": record["notes"], @@ -268,7 +267,7 @@ def _row_ids(cells): return sorted(cells) -def _write_cells(st, table_key, cells, *, by=amazon_profit.WRITER, via=amazon_profit.VIA): +def _write_cells(st, table_key, cells): import core.user_tables as ut def _set(cur): @@ -284,7 +283,7 @@ def _write_cells(st, table_key, cells, *, by=amazon_profit.WRITER, via=amazon_pr row[key] = value return cur - ut.update_with_activity(table_key, _set, st=st, by=by, via=via, + ut.update_with_activity(table_key, _set, st=st, by=amazon_profit.WRITER, via=amazon_profit.VIA, row_ids=_row_ids(cells)) @@ -875,11 +874,6 @@ def edit_project(project_id: str, body: Any = Body(default=None), session: Sessi def delete_project(project_id: str, session: Session = Depends(require_session)): """Delete one project and its targets.""" runtime = _writer(session) - record = _call(projects.get, runtime, project_id) - if record.get("type") == "pdp": - from routes_amazon_pdp import delete_workflow - delete_workflow(project_id, session=session) - return {"ok": True, "id": project_id} pairs = _target_pairs(runtime, project_id) _call(projects.delete, runtime, project_id, by=_by(session)) _repaint(runtime, pairs) diff --git a/api/routes_brand.py b/api/routes_brand.py index ead1b3b667e1ba96d49c21b4cf1f8f02f25fbd48..88c9b7dfa3c0598ea9ca1e303949b99638ceed8a 100644 --- a/api/routes_brand.py +++ b/api/routes_brand.py @@ -35,16 +35,13 @@ from __future__ import annotations import os import re -from hashlib import sha256 from xml.etree import ElementTree from pathlib import Path -from fastapi import APIRouter, Depends, Request -from fastapi.concurrency import run_in_threadpool -from fastapi.responses import FileResponse, Response +from fastapi import APIRouter, Depends +from fastapi.responses import FileResponse from deps import Session, err, require_session -from core import tenant_brand router = APIRouter(prefix="/api/v1") @@ -80,53 +77,6 @@ _MEDIA = {".png": "image/png", ".svg": "image/svg+xml"} _HERE = Path(__file__).resolve().parent -def _logo_call(fn, *args): - try: - return fn(*args) - except tenant_brand.LogoStorageError as error: - raise err(503, "logo_storage_unavailable", str(error)) from None - except ValueError as error: - raise err(422, "invalid_logo", str(error)) from None - - -def _logo_admin(session): - if not session.admin: - raise err(403, "admin_required", "Only an administrator can change the tenant logo.") - - -@router.get("/tenant-brand/logo") -def tenant_logo(session: Session = Depends(require_session)): - """Serve this tenant's logo to any signed-in user without changing state.""" - result = _logo_call(tenant_brand.logo, session.runtime) - if result is None: - raise err(404, "no_logo", "This workspace has no logo.") - data, mime = result - return Response(data, media_type=mime, headers={ - "ETag": f'"{sha256(data).hexdigest()}"', "Cache-Control": "private, no-cache", - "X-Content-Type-Options": "nosniff"}) - - -@router.put("/tenant-brand/logo") -async def put_tenant_logo(request: Request, session: Session = Depends(require_session)): - """Accept at most 2 MiB of raw PNG or JPEG bytes from an administrator.""" - _logo_admin(session) - data = bytearray() - async for chunk in request.stream(): - if len(data) + len(chunk) > tenant_brand.MAX_BYTES: - raise err(413, "logo_too_large", tenant_brand.SIZE_LIMIT) - data.extend(chunk) - return await run_in_threadpool( - _logo_call, tenant_brand.put_logo, session.runtime, bytes(data), session.uname) - - -@router.delete("/tenant-brand/logo", status_code=204) -def delete_tenant_logo(session: Session = Depends(require_session)): - """Remove this tenant's logo for an administrator.""" - _logo_admin(session) - _logo_call(tenant_brand.delete_logo, session.runtime) - return Response(status_code=204) - - def _roots(): env = (os.environ.get("AIOS_BRAND_DIR") or "").strip() out = [Path(env)] if env else [] diff --git a/api/routes_changes.py b/api/routes_changes.py index 05d0c274e2f7e52a5e2cd127e7440837fd768143..c790086a5f275d39f389d72bd294c2c1bba1d08b 100644 --- a/api/routes_changes.py +++ b/api/routes_changes.py @@ -62,7 +62,6 @@ import core.store as store import core.user_tables as user_tables import modules.customer_data as customer_data import modules.product_data as product_data -import virtual_sources from deps import Session, err, require_changes_session, require_session @@ -201,8 +200,6 @@ def _may_watch(session, scope): @router.get("/changes") def changes(scope: str = "", session: Session | None = Depends(require_changes_session)): """Serve legacy change checks without making Pg a resident client.""" - if virtual_sources.get(_clean_scope(scope)) is not None: - return {"scope": _clean_scope(scope), "tokens": {}, "backend": store.backend()} names = buckets_for(scope) if names is None: raise err(400, "bad_scope", "name the table you are watching, e.g. ?scope=customer") diff --git a/api/routes_competitors.py b/api/routes_competitors.py index 436cd676061454d0a78332a8515123b3a82cdc27..31f61ba1d56024cf954fae709a2fa7199f49b04a 100644 --- a/api/routes_competitors.py +++ b/api/routes_competitors.py @@ -39,19 +39,6 @@ def _open_mirror(runtime): cur.close() -@contextmanager -def _boundary_errors(): - try: - yield - except rivals.MirrorUnreadable as exc: - raise err(503, "amazon_mirror_unreadable", - f"The Amazon data could not be read ({exc}). Try again shortly.") from None - except rivals.BackendUnavailable: - raise err(503, "competitors_store_unavailable", "Competitors need the workspace database.") from None - except RuntimeError: - raise err(503, "store_not_ready", "Amazon data is still loading. Try again shortly.") from None - - def _rt(session): session.require(routes_amazon_profit.amazon_profit.TABLE_KEY) try: @@ -164,23 +151,6 @@ def competitors_list(grain: str, key: str, session: Session = Depends(require_se return _payload(runtime, grain, key, target, parent, routes_amazon_profit._may_edit(session), problem) -@router.get("/competitors/pool") -def competitors_pool(grain: str, key: str, session: Session = Depends(require_session)): - """Read the eligible Setup pool and its stored daily facts through the caller's tenant session.""" - runtime = _rt(session) - key = _grain(grain, key) - with _boundary_errors(), _open_mirror(runtime) as cur: - target, parent = _target(cur, grain, key) - if parent is not None: - raise err(400, "parent_has_no_rivals", rivals.PARENT_ROLLS_UP) - pool = rivals.pool(runtime, cur, target) - payload = {field: pool[field] for field in ('asin', 'productType', 'dataAsOf', 'week', 'month', - 'keywordRanksOn', 'cause', 'held', 'removed', 'competitors')} - if not payload['competitors']: - payload['cause'] = rivals.NO_POOL - return {**payload, 'canEdit': routes_amazon_profit._may_edit(session)} - - def _writable(session, body): runtime = _rt(session) if not routes_amazon_profit._may_edit(session): @@ -188,8 +158,16 @@ def _writable(session, body): body = body if isinstance(body, dict) else {} grain = body.get("grain") key = _grain(grain, body.get("key")) - with _boundary_errors(), _open_mirror(runtime) as cur: - target, parent = _target(cur, grain, key) + try: + with _open_mirror(runtime) as cur: + target, parent = _target(cur, grain, key) + except rivals.MirrorUnreadable as exc: + raise err(503, "amazon_mirror_unreadable", + f"The Amazon data could not be read ({exc}). Try again shortly.") from None + except rivals.BackendUnavailable: + raise err(503, "competitors_store_unavailable", "Competitors need the workspace database.") from None + except RuntimeError: + raise err(503, "store_not_ready", "Amazon data is still loading. Try again shortly.") from None if parent is not None: raise err(400, "parent_has_no_rivals", rivals.PARENT_ROLLS_UP) try: diff --git a/api/routes_connectors.py b/api/routes_connectors.py index 137e5138ff863bca305b0c80c3ac2b33055f5394..2c05f98064296b896834c77e0639b28d7975dee1 100644 --- a/api/routes_connectors.py +++ b/api/routes_connectors.py @@ -3315,6 +3315,9 @@ def _amazon_table_schedule(rt, connection_id=None, now=None): state = (landed.get("state") if landed.get("state") in ("refused", "failed", "throttled") else tick.get("state") or "never_run") reason = row_reason(table, state) + if state == "refused" and (reason or "").startswith("Amazon does not offer this report"): + # A report Amazon does not offer this seller is not available, never a failed sync. + state = "not_offered" # A source-internal mismatch published the table. Its warning stays until a clean reconcile. warning = _amazon_cause_text(saved.get(prefix + az.entity_key("warning_" + table)) or "") schedule.append({**row, "period": _amazon_table_period(table), @@ -3354,7 +3357,9 @@ def _amazon_sync_staleness(completed, schedule, now=None): fastest = max(amazon_client.CADENCE[table]["perDay"] for table in amazon_client.GROUP_A) allowed = 86400 / fastest + rel.STALE_GRACE_SECONDS - stamps = [row.get("lastSyncAt") for row in schedule if row["perDay"] == fastest] + # A not_offered row can carry an old watermark stamp. Amazon never sends that report, so it never ages the card. + stamps = [row.get("lastSyncAt") for row in schedule + if row["perDay"] == fastest and row.get("state") != "not_offered"] if not stamps: stamps = [completed] now = now or datetime.now(timezone.utc) diff --git a/api/routes_grid.py b/api/routes_grid.py index f519d9e8cb47f59af8f132ff2425575dfabed319..d942214b31b8cc0f76e609b9d8507f1cfb6f1041 100644 --- a/api/routes_grid.py +++ b/api/routes_grid.py @@ -20,7 +20,6 @@ from inspect import signature from fastapi import APIRouter, Body, Depends, Header, HTTPException, Query from deps import Session, err, module_gate, perms, require_session -import virtual_sources router = APIRouter(prefix="/api/v1") @@ -50,7 +49,6 @@ def row_patch_receipt(scope): raise err(400, 'bad_op_id', str(exc)) from exc session = bound.arguments['session'] scope_key = scope if isinstance(scope, str) else scope(bound.arguments) - virtual_sources.refuse_write(scope_key) bound.arguments['body'] = {key: value for key, value in body.items() if key != 'opId'} def apply(): @@ -235,7 +233,6 @@ def grid_choice_remove(scope: str = "customer", body: dict = Body(default=None), body = body if isinstance(body, dict) else {} scope = _scope_or_400(scope) - virtual_sources.refuse_write(scope) key = str(body.get("key") or "").strip()[:80] options = _cr.normalise(body.get("options") if isinstance(body.get("options"), list) else []) confirm = body.get("confirm") is True @@ -478,8 +475,6 @@ def workspace(scope: str = "customer", from routes_customers import grid_assembly scope = _scope_or_400(scope) - if virtual_sources.get(scope) is not None: - return {"workspace": virtual_sources.assembly(session, scope, with_rows=False)["workspace"]} # Wave 21 C4 — the per-scope gate the dependency no longer asserts: customer/cohort need # the customer module; product asserts PRODUCT_MODULE in its branch; ut_* needs only a # session (its wall is the table's own, inside the assembly). @@ -1529,7 +1524,6 @@ def bulk_cells(body: dict = Body(default=None), from routes_products import MODULE as PRODUCT_MODULE, scoped_pool scope = _scope_or_400((body or {}).get("scopeKey")) - virtual_sources.refuse_write(scope) if scope != "product": # Fail-closed with the reason: the shared stratum is a PRODUCT-topic mechanism today. raise err(400, "scope_not_bulk_writable", @@ -2132,7 +2126,6 @@ def grid_events_route(body: dict = Body(default=None), # typo degraded silently to customer-scope behaviour on a WRITE. Read and write must agree on # what a scope is, or the surface you read is not the surface you wrote. scope = _scope_or_400((body or {}).get("scopeKey")) - virtual_sources.refuse_write(scope) if scope.startswith("ut_") and any( isinstance(event, dict) and event.get("type") == "overlay_patch" for event in events): # Managed records require Route/Zone tools, including keys without stored definitions. diff --git a/api/routes_odoo_tables.py b/api/routes_odoo_tables.py index 3160d8b2afe92dc02eeff0bd8cfaa6cc8f3bc588..2967033ad9b0b55f0107550f789b4475a66d94fa 100644 --- a/api/routes_odoo_tables.py +++ b/api/routes_odoo_tables.py @@ -5096,12 +5096,10 @@ def _materialised_window(table_key, session, offset, limit, filters, filterConj, def _materialised_window_read(table_key, session, offset, limit, filters, filterConj, sorts, search, related, groupBy=None, groupKey=None, ids=None, cursor=None): from harness import datastore - import virtual_sources query = _selection_query({"filters": _json_arg(filters, "filters"), "filterConj": filterConj, "sorts": _json_arg(sorts, "sorts"), "search": search}) - virtual = virtual_sources.get(table_key) is not None - payload = virtual_sources.rows_payload(session, table_key, related=related) if virtual else None + payload = None window_offset = offset if table_key == 'ut_activity': if cursor is not None: @@ -5140,9 +5138,7 @@ def _materialised_window_read(table_key, session, offset, limit, filters, filter options = dict(select=spec['id'] if read.get('selection') else '_row_index', where=where, params=tuple(params), order_by=order or spec['id'], offset=window_offset, limit=limit or None, cur=read['cur'], **read['frm']) - if table_key == 'ut_activity' or virtual: - if virtual and cursor is not None: - raise err(400, 'window_cursor_unsupported', 'Continue this source with an offset.') + if table_key == 'ut_activity': win = datastore.window(**options) win['next_cursor'] = (read['payload'] or {}).get('next_cursor') else: @@ -5289,10 +5285,6 @@ def odoo_table_rows(table_key: str, "limit, filters, sorts and search in the query string.") ids = _record_ids(body) - import virtual_sources - if virtual_sources.get(table_key) is not None: - return _materialised_window_read(table_key, session, offset, limit, filters, filterConj, - sorts, search, related, groupBy, groupKey, ids=ids, cursor=cursor) # Admission resolves this Amazon database without publishing a process-wide binding. page = _page_defs(session) if str(table_key).startswith(("ut_amzn_", "ut_odoo_model_")) else None admitted = require_amazon_mirror(table_key, session, st=page[0] if page else None) diff --git a/api/routes_shares.py b/api/routes_shares.py index 22cee0bc521db77ac76e5c74ebc5627691f82283..a76f43418359cab15f7be4106d5b3833fc2be1d7 100644 --- a/api/routes_shares.py +++ b/api/routes_shares.py @@ -1221,8 +1221,6 @@ def get_cohort(scope: str, cohort_id: str, session: Session = Depends(require_se @router.get("/share/{kind}/{oid}") def get_share(kind: str, oid: str, session: Session = Depends(require_session)): kind = _kind_or_400(kind) - if kind == "report" and not session.admin: - raise err(404, "no_object", "no such item, or it is not shared with this account") if kind == "field": table_key, field_key = shares.split_field_oid(oid) if table_key and field_key: @@ -1281,9 +1279,8 @@ def get_share(kind: str, oid: str, session: Session = Depends(require_session)): **rec, "role": role, "mayAdminister": may_admin, - "maxGrantableRole": ( - "view" if kind == "report" else "edit" if claiming else shares.max_grantable_role( - kind, oid, session.uname, is_admin=_admin_share, st=session.runtime)), + "maxGrantableRole": "edit" if claiming else shares.max_grantable_role( + kind, oid, session.uname, is_admin=_admin_share, st=session.runtime), "ownerLabel": owner_label, # ⚠ WAVE 21 (C1 identity fix): grant entries BIND on USERNAMES, so the picker must carry # them. `assignable_people` serves bare display names because `user`-kind CELLS store @@ -1305,19 +1302,35 @@ def get_share(kind: str, oid: str, session: Session = Depends(require_session)): def _people(tenant): - """Active tenant identities, excluding test accounts, ordered by display name.""" - return sorted(users.assignable_identities(tenant), key=lambda p: p["name"].lower()) - - -def _report_entries_or_400(entries): - """A report grants download to named people alone.""" - for entry in entries: - if (not isinstance(entry, dict) or not str(entry.get("user") or "").strip() - or str(entry.get("user")).strip() == shares.EVERYONE): - raise err(400, "bad_entries", - "Add people by name. A report is shared one person at a time.") - if str(entry.get("role") or "view").strip().lower() != "view": - raise err(400, "bad_role", "A report grant can only allow download.") + """[{username, name}] for this tenant — same population as `assignable_people`, with the + BINDING identity alongside the display one. + + ⛔⛔ W49-T23 / D-235 — A BLANK TENANT RETURNS NOBODY, NOT EVERYBODY. This read + `if want and str(u.get('tenant') …) != want`, so a blank or `None` slug skipped the filter + entirely and handed one workspace every tenant's roster: the exact `if want and …` fail-OPEN + that W33-T37 closed in `core.users.assignable_people` / `avatar_map` one layer down, left + behind in the copy that lives here. The three callers all pass `session.tenant`, which is + `claims["t"]` and is already proven non-blank by `require_session` (an unresolvable slug is a + 401 before any route runs), so this changes no answer any live caller gets today — it removes + the shape, before a fourth caller arrives that does not carry that proof. Fail-closed, because + from this function a blank slug and "every tenant at once" are the same observation, and one + of the two callers below turns this roster into who may be GRANTED access. + """ + try: + reg = users.registry() or {} + except Exception: + return [] + want = str(tenant or '').strip().lower() + if not want: + return [] + out = [] + for uname, u in reg.items(): + if not isinstance(u, dict) or u.get('active') is False: + continue + if str(u.get('tenant') or 'royal-imports').strip().lower() != want: + continue + out.append({"username": str(uname), "name": str(u.get('name') or uname)}) + return sorted(out, key=lambda p: p["name"].lower()) def _stranger(session, kind, oid): @@ -1342,11 +1355,6 @@ def _put_share(kind, oid, body, session, *, _view_snapshot=None, _source_owner=N _notify=True): """Apply the shared share admission, ceiling and atomic writer for both doors.""" kind = _kind_or_400(kind) - if kind == "report" and not session.admin: - if _stranger(session, kind, oid): - raise err(404, "no_object", "no such item, or it is not shared with this account") - raise err(403, "not_owner", - "Only an administrator can change who can download this report.") if kind == "field": table_key, field_key = shares.split_field_oid(oid) if table_key and field_key: @@ -1420,8 +1428,6 @@ def _put_share(kind, oid, body, session, *, _view_snapshot=None, _source_owner=N raise err(400, "bad_entries", "entries must be a list of {user, role}. Send [] to un-share, which is how " "revoking is expressed") - if kind == "report": - _report_entries_or_400(entries) _entries_or_400(session, entries) # ⭐⭐ R4 / W40-T02 — THE CEILING. `may_administer` above now opens this door to an `edit` # grantee on a VIEW, so R4's other half ("a re-share may never exceed the role the re-sharer @@ -1617,8 +1623,7 @@ def _put_share(kind, oid, body, session, *, _view_snapshot=None, _source_owner=N try: out = shares.set_grants(kind, oid, entries, - owner=(None if kind == "report" else - session.uname if claiming else rec["owner"]), + owner=session.uname if claiming else rec["owner"], granter=session.uname, st=session.runtime, expect=rec["entries"]) diff --git a/api/routes_starred.py b/api/routes_starred.py index 93c34a74a2c936a4c1e468999425b0282d2690ad..c4a589d8da4228ddfb99b1cc3a09174fe45876c2 100644 --- a/api/routes_starred.py +++ b/api/routes_starred.py @@ -1231,9 +1231,6 @@ def starred_records(database: str = "", session: Session = Depends(require_sessi key = _clean_id(database) if not key: raise err(400, "bad_request", "no database was named") - import virtual_sources - if virtual_sources.get(key) is not None: - return {"database": key, "ids": [], "count": 0, "view": None} return _records_payload(session, key, _database_or_refuse(session, key)) @@ -1252,8 +1249,6 @@ def set_starred_record(body: dict = Body(default=None), raw_id = _clean_id(body.get("id")) if not key: raise err(400, "bad_request", "no database was named") - import virtual_sources - virtual_sources.refuse_write(key) if not raw_id.isdigit(): # Named rather than coerced: see `_record_ids` on why a non-numeric id cannot be stored. raise err(400, "bad_request", "a record id is a number") diff --git a/api/routes_tables.py b/api/routes_tables.py index bbf775f2cc661332ac92786b1a5763693aa958bd..3e6392232dcf77aec71b972287ea2a7b8fafab33 100644 --- a/api/routes_tables.py +++ b/api/routes_tables.py @@ -56,7 +56,6 @@ from fastapi import HTTPException, Request, Response from deps import Session, err, require_session from harness.runtime import DatastoreMismatch from routes_grid import row_patch_receipt -import virtual_sources router = APIRouter(prefix="/api/v1") @@ -286,8 +285,6 @@ def related_metrics(table_key: str, body: dict = Body(default=None), if (not isinstance(pin, dict) or not isinstance(pids, list) or any(type(pid) is not int or pid <= 0 for pid in pids)): raise err(400, "bad_metric_scope", "Name the origin record, relation and linked records.") - if virtual_sources.get(table_key) is not None: - return virtual_sources.related_metrics(session, table_key, pin, pids) session.require(table_key) viewer = copy.copy(session) viewer.runtime = _MetricReadRuntime(session.runtime) @@ -393,8 +390,6 @@ def _table_lend(runtime, table_key, defs_only=False): def _managed_or_refuse(session, table_key, *, mutation=False): """C1 generic-door policy. Dedicated Route/Zone handlers use their own capability.""" from core import route_system - if mutation: - virtual_sources.refuse_write(table_key) definition = _ut().get_one(table_key, st=session.runtime) # Route/Zone are virtual system descriptors until their first dedicated write, so looking # only in `user_tables` would leave a generic mutation path open during that empty state. @@ -589,7 +584,6 @@ def _defn_or_refuse(session, table_key, st=None, defs_only=False, scope_applied= proves nothing about which read produced it. `core.store.is_projected(defn)` is how a gate asserts the fast path was TAKEN. """ - virtual_sources.refuse_write(table_key) ut = _ut() # ⭐⭐ W31 QA — THE SWEEP, AND IT IS ONE LINE BECAUSE IT IS DONE HERE RATHER THAN PER ROUTE. # Owner: *"this is just one case, I need you to check and apply the fix everywhere too."* @@ -672,8 +666,8 @@ def _records_lock_message(table_key, definition, st): return f"This {name} is locked, so only user-created field values can be edited." -def _editor_or_refuse(session, table_key, st=None): - """Require editor authority on a database this session may see. `(defn, st)`.""" +def _records_or_refuse(session, table_key, st=None, *, cell_updates=None): + """Require editor authority and reserve locked database edits for authored field values.""" # Share one snapshot across the read, editor and record-mode checks. st = st if st is not None else _ut().lend(session.runtime) # ⭐ W36-T21 — `scope_applied=True` because every row write behind this wall is bounded by a @@ -683,12 +677,6 @@ def _editor_or_refuse(session, table_key, st=None): defn = _defn_or_refuse(session, table_key, st=st, scope_applied=True) if not _ut().may_edit_database(table_key, session.uname, is_admin=session.admin, st=st): raise err(403, "database_read_only", "Can edit access is required to change this database.") - return defn, st - - -def _records_or_refuse(session, table_key, st=None, *, cell_updates=None): - """Require editor authority and reserve locked database edits for authored field values.""" - defn, st = _editor_or_refuse(session, table_key, st) if not _ut().records_mutable(table_key, st=st): if cell_updates and table_key != "ut_activity": from core import shared_overlay @@ -1063,9 +1051,6 @@ def _activity_grid_rows(session): @router.get('/tables/{table_key}/activity') def table_activity(table_key: str, cursor: str = None, limit: int = 50, session: Session = Depends(require_session)): - if virtual_sources.get(table_key) is not None: - from core import activity_log - return {'records': [], 'next_cursor': None, 'fields': activity_log.definition()['fields']} if table_key.startswith('ut_'): _defn_or_refuse(session, table_key, defs_only=True, scope_applied=True) elif table_key not in ('customer_data', 'product_data'): @@ -2137,9 +2122,6 @@ def _scoped_table_rows(session, table_key, st, *, cursor=None, limit=None): def ut_assembly(session: Session, table_key: str, storage_key: str = "", consume_corrections: bool = False, with_rows: bool = True, st=None, *, fields_only: bool = False, needed=None, cursor=None, limit=None): - if virtual_sources.get(table_key) is not None: - return virtual_sources.assembly(session, table_key, with_rows=with_rows and not fields_only, - storage_key=storage_key) from core import grid_events actor = grid_events.request_admission(session, table_key + '_table_workspace') result = _ut_assembly(actor, table_key, storage_key, consume_corrections, with_rows, st, @@ -3365,8 +3347,6 @@ def table_rows(table_key: str, session: Session = Depends(require_session), (found by this route's own gate check, not by luck).""" import aios_grid - if virtual_sources.get(table_key) is not None: - return virtual_sources.rows_payload(session, table_key, related=related) if table_key == 'ut_activity': if related: raise err(409, 'activity_related_unavailable', 'Open the Activity database to read events.') @@ -4541,8 +4521,7 @@ def restore_history(table_key: str, event_id: str, body: dict = Body(default=Non from core import grid_events _managed_or_refuse(session, table_key, mutation=True) - # A locked database admits authored values only, so its record wall waits for the retained values. - _editor_or_refuse(session, table_key) + _records_or_refuse(session, table_key) body = body if isinstance(body, dict) else {} row_id = str(body.get("rowId") or "").strip() if not row_id: @@ -4572,7 +4551,6 @@ def restore_history(table_key: str, event_id: str, body: dict = Body(default=Non raise err(404, "history_event_not_found", "that retained value version is unavailable") if any(key in (g.get("hidden") or ()) for key in updates): raise err(403, "field_hidden", "that retained version includes a hidden field") - _records_or_refuse(session, table_key, cell_updates=updates) prior_history_ids = { str(item.get("id")) for item in _ut().value_history( table_key, row_id, st=session.runtime, retention_days=days, diff --git a/api/sync_runtime.py b/api/sync_runtime.py index db7d5927f779bc768646f10fefb52a81b74c7f1a..f6e99682593cd01dfd19dd1561f592a6e5838df3 100644 --- a/api/sync_runtime.py +++ b/api/sync_runtime.py @@ -6,7 +6,7 @@ publish is 878.8 s inside the 1,200 s limit, so a pass fits one job. Decision (lead ruling, 2026-11-03): the default job is RESIDENT (`sync_worker.py --job`). It runs every worker loop as the old child did, because about ten heavy loops (automation, Amazon ads, inbox, payouts, aged stock, Sheet refresh, restock, returns, materialization, first-load floor) -live only in the worker. A resident job hands over to a new job only at a pass boundary of the +live only in the worker. A resident job hands over to a new job only at an idle point of the resync loop, when the web's deployed version differs from its own or under 3 h of its 24 h timeout remain. AIOS_SYNC_JOB_MODE=per-pass keeps one job per pass (`sync_worker.py --once`). The job runs the Space's own image (hf.co/spaces/), proved by HF job @@ -17,7 +17,7 @@ AIOS_SYNC_RUNTIME picks the runtime: `hf-job`, `external` (a separately run resi HF sets SPACE_HOST in a Space. The DigitalOcean staging config sets SPACE_ID only. Web side: `tick` submits a job for each tenant with no live job: always when resident, only when -its work is due when per-pass. Worker side: `relay_at_boundary` hands queued work to the store. +its work is due when per-pass. Worker side: `idle_wait` is the resync loop's only exit point. The tenant's C-status `sync_worker` entry carries `job` {id, submittedAt, image, tenant}, `schedule` {nextPassAt, at, reason} and `web` {version, at}. Stamps are ISO-8601 UTC, the format of RESYNC_STATE's. @@ -29,7 +29,6 @@ import math import os import threading import time -from copy import deepcopy RUNTIMES = ("hf-job", "external", "off") RUNTIME_ENV = "AIOS_SYNC_RUNTIME" @@ -234,51 +233,8 @@ def relay_due(now=None): return "" -def _handoff_queued_requests(): - """Persist ordinary queued receipts without changing durable events or interrupting a claim.""" - from core import connector_sync - import process_role - with connector_sync._SYNC_REQUEST_LOCK: - receipts = deepcopy(connector_sync._SYNC_REQUESTS) - runtimes = dict(connector_sync._SYNC_REQUEST_RUNTIMES) - if any(row.get("state") == "running" for row in receipts.values()): - return None - try: - for row in receipts.values(): - if row.get("state") != "queued" or row.get("tables") is not None: - continue - tenant = row["tenant"] - rt = runtimes.get(tenant) or process_role._status_runtime(tenant) - if rt.key != tenant: - raise ValueError("The relay desk does not match the receipt tenant.") - connector_sync.queue_remote_request(rt, row, row.get("connectionId")) - except Exception as exc: - print(f"[sync-worker] job: relay handoff failed ({type(exc).__name__}); keeping this job running", flush=True) - return None - return receipts - - -def relay_at_boundary(next_at=None, *, exit_fn=None): - """Relay a resident job after its current write and projection finish, before another claim.""" - from core import connector_sync - if not ONCE["on"] or ONCE["mode"] != "resident": - return False - reason = relay_due() - if not reason: - return False - receipts = _handoff_queued_requests() - if receipts is None: - return False - with connector_sync._SYNC_REQUEST_LOCK: - # No receipt can be admitted or claimed between the final handoff check and exit. - if connector_sync._SYNC_REQUESTS != receipts: - return False - finish(time.time() if next_at is None else next_at, reason, exit_fn=exit_fn) - return True - - def idle_wait(event, seconds, *, exit_fn=None): - """Wait between passes, relaying a resident job before a queued wake can start another pass.""" + """The resync loop's wait, and a job's only exit point. Outside a job it is `event.wait(seconds)`.""" if not ONCE["on"]: return event.wait(seconds) if seconds is None or not math.isfinite(float(seconds)): @@ -286,21 +242,18 @@ def idle_wait(event, seconds, *, exit_fn=None): deadline = time.time() + max(0.0, min(float(seconds), MAX_IDLE_SECONDS)) resident = ONCE["mode"] == "resident" while True: - if resident and relay_at_boundary(deadline, exit_fn=exit_fn): - return False remaining = deadline - time.time() if remaining <= IDLE_GRACE_SECONDS: return event.wait(max(0.0, remaining)) # Per-pass: the grace lets the desk poll admit a request queued while the pass ran. if event.wait(min(RELAY_CHECK_SECONDS if resident else IDLE_GRACE_SECONDS, remaining)): - if resident and relay_at_boundary(deadline, exit_fn=exit_fn): - return False return True - if resident: + reason = relay_due() if resident else "idle" + if not reason: continue busy, wake_at = _desk_state(time.time()) if not busy: - finish(deadline if wake_at is None else min(deadline, wake_at), "idle", exit_fn=exit_fn) + finish(deadline if wake_at is None else min(deadline, wake_at), reason, exit_fn=exit_fn) return False @@ -345,7 +298,7 @@ def no_lease_line(tenant, seconds=None): def _pass_in_flight(): - """True while the store resync loop runs and owns its pass boundaries.""" + """True while the store resync loop runs: it relays at its own idle point.""" return any(t.name == "store-resync" and t.is_alive() for t in threading.enumerate()) @@ -356,7 +309,9 @@ def run_once(*, exit_fn=None, alive=None): if alive(): continue if ONCE["mode"] == "resident": - if relay_at_boundary(exit_fn=exit_fn): + reason = relay_due() + if reason and not _desk_state(time.time())[0]: + finish(time.time(), reason, exit_fn=exit_fn) return False continue finish(time.time() + MAX_IDLE_SECONDS, diff --git a/api/test_odoo_rank_refresh.py b/api/test_odoo_rank_refresh.py index 8c42287136c18dc6c87680a9ab3f4e5d86ac45bd..c928eff5c133f2f0e1d49b33f66fb74ac27c2060 100644 --- a/api/test_odoo_rank_refresh.py +++ b/api/test_odoo_rank_refresh.py @@ -1059,7 +1059,7 @@ class RankRefreshTests(unittest.TestCase): def run_slot_loop(self, memory, *, boot, max_sleeps, tenant='royal-imports', config=None, reports=None, checkpoints=None, take=None, slot_flags=None, status=None, - stop_at=None): + stop_at=None, on_pass=None): """Drive the real loop and its real slot resolver on a clock that advances at each wait.""" import odoo_relational as rel source = ast.parse(Path(__file__).with_name('main.py').read_text(encoding='utf-8')) @@ -1086,6 +1086,8 @@ class RankRefreshTests(unittest.TestCase): def sync(manual_flag, passes, entities): events.append(('mirror', now[0], manual_flag)) + if on_pass is not None: + on_pass(store, now[0]) if manual_flag: return None, None, None if reports: @@ -1684,5 +1686,72 @@ class RankRefreshTests(unittest.TestCase): self.assertEqual((memory['slotResumedFor'], memory['slotResumes']), (slot_at, 1)) + def test_h1_freshness_reads_the_keychain_default_slot(self): + """H1: a pass written under the keychain default id reads fresh; no id keeps the default slot""" + from datetime import datetime, timezone + import odoo_relational as rel + from core import connector_sync + now = datetime(2026, 11, 18, 9, 0, tzinfo=timezone.utc) + good = '2026-11-18T07:32:38+00:00' + entry = {'lastGoodSyncAt': good, 'resync': {'connectionId': 'k_x'}, + 'pass': {'startedAt': '2026-11-18T07:00:00+00:00', 'endedAt': good, + 'publishedAt': good, 'outcome': 'held'}} + keyed = self.SlotStore('acme', {connector_sync.STATUS_KEY: {'odoo': {'k_x': dict(entry)}}}) + with patch.object(rel, 'default_connection_id', return_value='k_x'): + fresh = rel.sync_freshness(keyed, now=now) + self.assertFalse(fresh['stale']) + self.assertEqual(fresh['lastGoodSyncAt'], good) + self.assertEqual((fresh['pass']['outcome'], fresh['resync']), ('held', {'connectionId': 'k_x'})) + # Control: read through the "default" slot alone, the same store is stale (the live defect). + with patch.object(rel, 'default_connection_id', return_value=None): + self.assertTrue(rel.sync_freshness(keyed, now=now)['stale']) + # Negative control: an empty keychain slot, or no keychain id, still reads the "default" slot. + plain = self.SlotStore('acme', {connector_sync.STATUS_KEY: {'odoo': {'default': dict(entry)}}}) + for cid in (None, 'k_x'): + with patch.object(rel, 'default_connection_id', return_value=cid): + fresh = rel.sync_freshness(plain, now=now) + self.assertFalse(fresh['stale'], cid) + self.assertEqual(fresh['lastGoodSyncAt'], good, cid) + + def test_h1_a_held_pass_that_published_records_its_success(self): + """H1: a held pass that published stamps lastSuccessAt and the source stage; a refused one writes nothing""" + import odoo_relational as rel + import routes_connectors + from core import connector_sync + last = self.utc('2026-10-22T06:05:00Z') + boot = self.utc('2026-10-24T07:00:00Z') + memory = {'passAt': last, 'fullRebuildAt': last, 'fingerprints': {'res_partner': (3, 7)}} + held = {'account_move_line': {'phase': 'backfill', 'pulled': 0}} + marker = ('account_move_line.cols.amount_residual', 'running', 500, None, 100_000) + + def publication(offset): + def write(store, at): + row = {'startedAt': connector_sync._utc_stamp(at), 'outcome': 'running'} + if offset is not None: + row.update(outcome='done', publishedAt=connector_sync._utc_stamp(at + offset)) + store.docs[connector_sync.STATUS_KEY] = {'odoo': {'k_x': {'pass': row}}} + return write + # Published during this pass; published by an earlier pass; publication refused. + for offset, published in ((5, True), (-3600, False), (None, False)): + stage = SimpleNamespace(calls=[]) + with patch.object(rel, 'default_connection_id', return_value='k_x'), \ + patch.object(routes_connectors, 'record_odoo_stage', + lambda *a, **k: stage.calls.append((a, k))): + self.run_slot_loop(memory, boot=boot, max_sleeps=1, reports=[held], + checkpoints=[(marker,), (marker,)], on_pass=publication(offset)) + self.assertTrue(self.slot_state['publicationHeld'], offset) + self.assertEqual(self.slot_state['connectionId'], 'k_x') + self.assertIn('publication held; no vendor pulls or relational rebuild', self.loop_output) + if published: + self.assertEqual(self.slot_state.get('lastSuccessAt'), self.slot_state['lastPassAt']) + self.assertEqual(len(stage.calls), 1) + args, kwargs = stage.calls[0] + self.assertEqual(args[1:], ('k_x', 'source', 'ready')) + self.assertEqual(kwargs['at'], connector_sync._utc_stamp(boot + 5)) + else: + self.assertNotIn('lastSuccessAt', self.slot_state, offset) + self.assertEqual(stage.calls, [], offset) + + if __name__ == '__main__': unittest.main() diff --git a/api/usage_ledger.py b/api/usage_ledger.py index a806ca8c4243d861de182ab625fd3916fbb92eaa..4d42c06bc5bf7e6f06bddf11fbc908bf05a8be34 100644 --- a/api/usage_ledger.py +++ b/api/usage_ledger.py @@ -3,13 +3,16 @@ R9: *"ONE meter for every AI surface. The dashboard REPORTS weekly usage against an allowance and does NOT cut anyone off this wave; over the allowance it still works, shown red."* -⛔ THIS FILE MEASURES AND NEVER ENFORCES. Token allowances remain report-only. -Image-making limits live in `image_budget`, which reads measured image dollars and its own holds. +⛔ SO THIS FILE MEASURES AND NEVER ENFORCES. There is no ceiling here, no refusal, and no caller of +this module may branch on its answer. The one limit the product does enforce is per COLUMN and lives +where it was built (`ai_enrich.ceiling_report`); this is the tenant-wide REPORT, and mixing the two +would make an over-allowance week silently stop somebody's enrichment run. -**THE SURFACES.** `assistant` (a person asking the Assistant a question) · `field_agent` (an AI +**THE FOUR SURFACES.** `assistant` (a person asking the Assistant a question) · `field_agent` (an AI enrichment column filling cells) · `ai_review` (a model deciding a review stage) · `automation_draft` -(a sentence becoming a draft flow) · `image_gen` (image making, edits and supporting reads or briefs). -The gate in `verify_api.py` derives LLM entry points from the code rather than this list. +(a sentence becoming a draft flow). They are the four LLM entry points the product has, and the gate +in `verify_api.py` DERIVES them from the code rather than reading this list, so a fifth entry point +is caught by the gate and not by somebody remembering to edit a tuple. ⛔⛔ **`record()` MUST BE HANDED ITS TARGET, AND THAT IS A MEASUREMENT, NOT A STYLE CHOICE.** The obvious design is to bind `(runtime, username)` into a `contextvars.ContextVar` inside @@ -29,17 +32,17 @@ that hides the gap is worse than one that admits it. """ from __future__ import annotations -from decimal import Decimal, InvalidOperation, ROUND_HALF_UP import os import time -#: Weekly counters and image-settlement retry receipts contain no prompts or generated content. +#: The store bucket. Small by construction — counters only, never a per-call log: a log would grow +#: without bound on a route that runs per row, and nothing in R9 asks a question a log answers. STORE_KEY = "ai_usage" -#: The AI surfaces (C7). ⚠ A surface this list does not name is REFUSED rather than stored under +#: The four AI surfaces (C7). ⚠ A surface this list does not name is REFUSED rather than stored under #: whatever string arrived, because a typo'd surface is a meter that reads low forever with a second #: row nobody looks at. -SURFACES = ("assistant", "field_agent", "ai_review", "automation_draft", "image_gen") +SURFACES = ("assistant", "field_agent", "ai_review", "automation_draft") #: The weekly allowance the dashboard measures against (R9). ⛔ IT IS A REPORTING LINE, NOT A GATE. #: Env-tunable so a deployment can set a real number without a release. @@ -129,39 +132,26 @@ def _int_or_none(value): return None -def usd_micros(value): - """Return nonnegative dollars as whole micro-dollars, rounded half up, or None if unknown.""" - if isinstance(value, bool) or not isinstance(value, (int, float, str, Decimal)): - return None - try: - amount = Decimal(str(value)) - if amount.is_finite() and amount >= 0: - return int((amount * 1_000_000).to_integral_value(rounding=ROUND_HALF_UP)) - except (InvalidOperation, ValueError, OverflowError): - return None - return None - - -def _book_dollars(row, micros, calls, entry_id): - """Book measured dollars or a separate unknown count, resolving a single-call receipt once.""" - receipts = row.setdefault("_cost_entries", {}) if entry_id else {} - repeated = bool(entry_id and entry_id in receipts) - if repeated and (receipts[entry_id] is not None or micros is None): - return False - if repeated: - row["usd_unmeasured"] = max(0, int(row.get("usd_unmeasured") or 0) - 1) - if micros is None: - row["usd_unmeasured"] = int(row.get("usd_unmeasured") or 0) + calls - else: - row["usd_micros"] = int(row.get("usd_micros") or 0) + micros - if entry_id: - receipts[entry_id] = micros - return not repeated - - def record(surface, provider, model, tokens_in=None, tokens_out=None, *, - total=None, calls=1, cost_usd=None, st=None, user="", week=None, entry_id=""): - """Report a call, with optional held week and single-call retry id, without enforcing a limit.""" + total=None, calls=1, st=None, user=""): + """Book ONE AI call. Returns True when the line landed durably, False when it could not. + + C7's signature is the five positional parameters; everything after `*` is this module's own + extension and each one has a reason: + + · `total` — for a caller that knows the total and not the split (`ai_enrich.run_field` + accumulates a per-run total through an `_ask` contract a gate injects against, and changing + that contract to carry a split would break the injected shape for no gain here). + · `calls` — so a RUN over many rows books one line with its real call count instead of one + store write per row. + · `st` / `user` — the target. Required in practice; see this module's header for the measured + reason a contextvar cannot supply them. + + ⛔ IT NEVER RAISES. A meter that can break the feature it measures is worse than no meter: a + store blip during an enrichment run would abort the run and lose the cells. + ⛔ AND IT NEVER SILENTLY SUCCEEDS. A call it cannot attribute increments `UNATTRIBUTED`, which + `GET /usage` shows an admin. + """ key = str(surface or "").strip() if key not in SURFACES: # Refused rather than stored: a surface nobody named is a row nobody reads, and the meter @@ -172,16 +162,15 @@ def record(surface, provider, model, tokens_in=None, tokens_out=None, *, if tot is None and (ins is not None or outs is not None): tot = (ins or 0) + (outs or 0) n = max(0, int(calls) if isinstance(calls, int) and not isinstance(calls, bool) else 1) - if not n or (entry_id and n != 1): + if not n: return False uname = str(user or "").strip().lower() if st is None: UNATTRIBUTED["calls"] += n UNATTRIBUTED["surfaces"][key] = UNATTRIBUTED["surfaces"].get(key, 0) + n return False - week = week or week_of() + week = week_of() prov, mdl = str(provider or "")[:40], str(model or "")[:80] - micros = usd_micros(cost_usd) def _up(data): data = data if isinstance(data, dict) else {} @@ -189,16 +178,18 @@ def record(surface, provider, model, tokens_in=None, tokens_out=None, *, row = ((weeks.setdefault(week, {}) .setdefault(uname or "-", {})) .setdefault(key, {})) - if _book_dollars(row, micros, n, entry_id): - row["calls"] = int(row.get("calls") or 0) + n - if tot is None: - row["unmeasured"] = int(row.get("unmeasured") or 0) + n - else: - row["tokens"] = int(row.get("tokens") or 0) + tot - if ins is not None: - row["tokens_in"] = int(row.get("tokens_in") or 0) + ins - if outs is not None: - row["tokens_out"] = int(row.get("tokens_out") or 0) + outs + row["calls"] = int(row.get("calls") or 0) + n + if tot is None: + # ⛔ "the call happened, the tokens are unknown" IS A STATE, and it has to be + # representable or the meter quietly under-reports. The dashboard shows it as a count of + # unmeasured calls beside the token total. + row["unmeasured"] = int(row.get("unmeasured") or 0) + n + else: + row["tokens"] = int(row.get("tokens") or 0) + tot + if ins is not None: + row["tokens_in"] = int(row.get("tokens_in") or 0) + ins + if outs is not None: + row["tokens_out"] = int(row.get("tokens_out") or 0) + outs if prov: row["provider"] = prov if mdl: @@ -210,8 +201,12 @@ def record(surface, provider, model, tokens_in=None, tokens_out=None, *, return weeks try: - # Image dollars must persist before their budget holds can disappear. - st.update(STORE_KEY, _up, flush="sync" if key == "image_gen" else "async") + # ⚠ `flush='async'` — the coalescing mode ([[store-async-flush]]). This is the hot path of + # every AI surface in the product (an enrichment run books one line per RUN, but the + # assistant books one per question), and a blocking hub upload per call would put a network + # round trip inside the answer a person is waiting for. The mutation applies to the + # in-process cache immediately, so `GET /usage` reads its own writes. + st.update(STORE_KEY, _up, flush="async") return True except Exception: # noqa: BLE001 UNATTRIBUTED["calls"] += n @@ -220,8 +215,7 @@ def record(surface, provider, model, tokens_in=None, tokens_out=None, *, def _blank_row(): - return {"calls": 0, "tokens": 0, "tokens_in": 0, "tokens_out": 0, "unmeasured": 0, - "usd_micros": 0, "usd_unmeasured": 0} + return {"calls": 0, "tokens": 0, "tokens_in": 0, "tokens_out": 0, "unmeasured": 0} def _fold(rows): @@ -229,7 +223,7 @@ def _fold(rows): for row in rows: if not isinstance(row, dict): continue - for field in out: + for field in ("calls", "tokens", "tokens_in", "tokens_out", "unmeasured"): got = row.get(field) if isinstance(got, int) and not isinstance(got, bool): out[field] += max(0, got) diff --git a/platform/core/amazon_projects.py b/platform/core/amazon_projects.py index ac296c76ca935eac1d25e60a4a7e2f6b27ea3673..3fddd9b8abcff538be82cb16772ae3f129a7eaac 100644 --- a/platform/core/amazon_projects.py +++ b/platform/core/amazon_projects.py @@ -26,7 +26,6 @@ STATUSES = ('Backlog', 'Planned', 'In progress', 'Completed', 'Canceled') COMPLETED = 'Completed' CANCELED = 'Canceled' PRIORITIES = ('No priority', 'Urgent', 'High', 'Medium', 'Low') -TYPES = ('project', 'pdp') #: Health turns At risk this many days before the target date. AT_RISK_DAYS = 7 CHUNK = 200 @@ -244,8 +243,6 @@ def _record(values, columns=PROJECT_COLUMNS): value = values[len(columns) + 3] if len(values) > len(columns) + 3 else None members = json.loads(value) if isinstance(value, str) else value result['contributors'] = sorted(members or [], key=lambda member: member['username']) - result['type'] = values[len(columns) + 4] if len(values) > len(columns) + 4 else 'project' - result['listed'] = bool(values[len(columns) + 5]) if len(values) > len(columns) + 5 else True return result @@ -751,53 +748,11 @@ def _activity_target_preview(value, truncated): return f'{shown}. Target list incomplete' if shown else 'Target list incomplete' -def _pdp_item_stages(value, pdp): - """Accept complete typed workflow item identities before comparing historical stages.""" - if not isinstance(value, dict) or value.get('type') != 'pdp' or not isinstance(value.get('items'), list): - return None - stages = {} - for item in value['items']: - if (not isinstance(item, dict) or not isinstance(item.get('asin'), str) - or not isinstance(item.get('stage'), str) or item['stage'] not in pdp.STAGE_PROGRESS - or item['asin'] in stages): - return None - stages[item['asin']] = item['stage'] - return stages - - -def _pdp_activity_facts(event, field, before, after): - """Project only C4 action facts retained in the bounded stored event.""" - from core import amazon_pdp as pdp - if event.get('via') != pdp.VIA: - return None - unavailable = {'action': 'unavailable'} - if event.get('changes_truncated'): - return unavailable - old, new = (_pdp_item_stages(value, pdp) for value in (before, after)) - if field == 'setup' and before is None and new is not None: - scope = after.get('scope') - scope = scope if isinstance(scope, list) and scope and all(key in pdp.SCOPE_PARTS for key in scope) else None - return {'action': 'created', 'scope': scope} - if old is not None and new is not None: - changed = {stage for asin, stage in new.items() if asin in old and old[asin] != stage} - if len(changed) == 1: - stage = next(iter(changed)) - if field == ('publish' if stage == 'published' else stage): - return {'action': 'moved', 'stage': stage} if stage != 'published' else {'action': 'published'} - return {'action': 'setup'} if field in ('setup', 'competitors') and not changed else unavailable - if isinstance(after, dict) and isinstance(after.get('slot'), str) and after['slot'] in pdp.SLOTS: - part = pdp.SLOTS[after['slot']]['key'] - if after.get('part') == part and field in ('draft', 'approve', 'request'): - return {'action': {'draft': 'saved', 'approve': 'approved', 'request': 'requested'}[field]} - return {'action': 'ai'} if field == 'ai' else unavailable - - def _activity_entries(event, actor_names, task_names): """Project one log event into named activity entries.""" out = [] for index, (field, change) in enumerate((event.get('changes') or {}).items()): before, after = change.get('before'), change.get('after') - pdp_facts = _pdp_activity_facts(event, field, before, after) if field == 'project': action, field = ('created' if before is None else 'deleted'), None elif field == 'targets': @@ -820,8 +775,6 @@ def _activity_entries(event, actor_names, task_names): 'by': event['by'], 'byUsername': event['by'], 'byName': actor_names.get(event['by'], event['by']), 'action': action, 'field': field, 'from': shown[0], 'to': shown[1]}) - if pdp_facts is not None: - out[-1]['pdp'] = pdp_facts return out @@ -1142,8 +1095,6 @@ class PgProjectStore: notes TEXT, completed_on DATE, source_id TEXT, created_at TEXT NOT NULL, updated_at TEXT NOT NULL, created_by TEXT, rollup JSONB, impact JSONB, - type TEXT NOT NULL DEFAULT 'project' CHECK (type IN ({_in_sql(TYPES)})), - listed BOOLEAN NOT NULL DEFAULT TRUE, PRIMARY KEY (tenant, id), UNIQUE (tenant, source_id), CHECK ((status = '{COMPLETED}' AND completed_on IS NOT NULL) OR (status <> '{COMPLETED}' AND completed_on IS NULL)))''') @@ -1161,11 +1112,8 @@ class PgProjectStore: # One row's lookup is one indexed read on this index. con.execute(f'CREATE INDEX IF NOT EXISTS "amazon_project_targets_entity" ' f'ON {self.t(TARGETS)} (tenant, grain, key, project_id)') - additions = {'rollup': 'JSONB', 'impact': 'JSONB', - 'type': f"TEXT NOT NULL DEFAULT 'project' CHECK (type IN ({_in_sql(TYPES)}))", - 'listed': 'BOOLEAN NOT NULL DEFAULT TRUE'} - for field, declaration in additions.items(): - con.execute(f'ALTER TABLE {self.t(PROJECTS)} ADD COLUMN IF NOT EXISTS {field} {declaration}') + for field in ('rollup', 'impact'): + con.execute(f'ALTER TABLE {self.t(PROJECTS)} ADD COLUMN IF NOT EXISTS {field} JSONB') @contextmanager def _write(self): @@ -1178,20 +1126,13 @@ class PgProjectStore: cols = ', '.join(f'p.{c}' for c in PROJECT_COLUMNS) count = (f'(SELECT count(*) FROM {self.t(TARGETS)} AS c ' 'WHERE c.tenant = p.tenant AND c.project_id = p.id)') - return (f'SELECT {prefix}{cols}, {count}, {self._rollup_projection()}, {members}, ' - f'{self._type_projection()}, {self._listed_projection()} ' + return (f'SELECT {prefix}{cols}, {count}, {self._rollup_projection()}, {members} ' f'FROM {self.t(PROJECTS)} AS p WHERE p.tenant = %s{where}') def _rollup_projection(self): # Old schemas read blank figures until a guarded write adds the columns. return "to_jsonb(p)->'rollup', to_jsonb(p)->'impact'" - def _type_projection(self): - return "COALESCE(to_jsonb(p)->>'type', 'project')" - - def _listed_projection(self): - return "COALESCE((to_jsonb(p)->>'listed')::boolean, TRUE)" - def _members_projection(self, con): if con.execute('SELECT to_regclass(%s)', (self.t(MEMBERS),)).fetchone()[0] is None: return 'NULL' @@ -1237,19 +1178,15 @@ class PgProjectStore: 'WHERE tenant = %s AND id = %s', (stamp, json.dumps(impact), self.tenant_slug, project_id)) - def _insert(self, con, record, pairs, stamp): - columns = (*PROJECT_COLUMNS, 'impact', 'type', 'listed') - values = (*[record[c] for c in PROJECT_COLUMNS], - json.dumps(_with_result_configs(None, record.get('result_configs', []))), - record.get('type', 'project'), record.get('listed', True)) - con.execute(f'INSERT INTO {self.t(PROJECTS)} ({", ".join(columns)}) ' - f'VALUES ({_marks(len(PROJECT_COLUMNS))}, %s::jsonb, %s, %s)', values) - self._insert_targets(con, record['id'], pairs, stamp) - self._replace_members(con, record['id'], record.get('contributors', []), [], stamp) - def insert(self, record, pairs, stamp): with self._write() as con: - self._insert(con, record, pairs, stamp) + columns = (*PROJECT_COLUMNS, 'impact') + values = (*[record[c] for c in PROJECT_COLUMNS], + json.dumps(_with_result_configs(None, record.get('result_configs', [])))) + con.execute(f'INSERT INTO {self.t(PROJECTS)} ({", ".join(columns)}) ' + f'VALUES ({_marks(len(columns) - 1)}, %s::jsonb)', values) + self._insert_targets(con, record['id'], pairs, stamp) + self._replace_members(con, record['id'], record['contributors'], [], stamp) return self._one(con, record['id']) def get(self, project_id): @@ -1261,8 +1198,6 @@ class PgProjectStore: def update(self, project_id, changes, stamp): with self._write() as con: current = self._one(con, project_id, lock=True) - if current['type'] == 'pdp' and set(changes) & {'progress', 'status', 'completed_on'}: - raise ValueError('PDP item stages control project progress and completion.') candidate = _fields(changes, previous=current) if 'contributors' in changes: candidate['contributors'] = _member_values(changes['contributors'], self.tenant_slug) @@ -1278,25 +1213,23 @@ class PgProjectStore: self._replace_members(con, project_id, candidate['contributors'], current['contributors'], stamp) return self._one(con, project_id) - def _delete(self, con, project_id): - for table in (MEMBERS, TARGETS): - con.execute(f'DELETE FROM {self.t(table)} WHERE tenant = %s AND project_id = %s', - (self.tenant_slug, project_id)) - row = con.execute(f'DELETE FROM {self.t(PROJECTS)} WHERE tenant = %s AND id = %s RETURNING id', - (self.tenant_slug, project_id)).fetchone() - if row is None: - raise KeyError('Project not found.') - def delete(self, project_id): with self._write() as con: - self._delete(con, project_id) + con.execute(f'DELETE FROM {self.t(MEMBERS)} WHERE tenant = %s AND project_id = %s', + (self.tenant_slug, project_id)) + con.execute(f'DELETE FROM {self.t(TARGETS)} WHERE tenant = %s AND project_id = %s', + (self.tenant_slug, project_id)) + row = con.execute(f'DELETE FROM {self.t(PROJECTS)} WHERE tenant = %s AND id = %s RETURNING id', + (self.tenant_slug, project_id)).fetchone() + if row is None: + raise KeyError('Project not found.') return {'deleted': project_id} def list_all(self): with self._connection() as con: if not self._exists(con): return [] - rows = con.execute(self._select_sql(f' AND {self._listed_projection()} ORDER BY p.created_at, p.id', + rows = con.execute(self._select_sql(' ORDER BY p.created_at, p.id', members=self._members_projection(con)), (self.tenant_slug,)).fetchall() return [_record(row) for row in rows] @@ -1320,12 +1253,10 @@ class PgProjectStore: count = (f'(SELECT count(*) FROM {self.t(TARGETS)} AS c ' 'WHERE c.tenant = p.tenant AND c.project_id = p.id)') rows = con.execute( - f'SELECT t.grain, t.key, {cols}, {count}, {self._rollup_projection()}, {members}, ' - f'{self._type_projection()}, {self._listed_projection()} ' + f'SELECT t.grain, t.key, {cols}, {count}, {self._rollup_projection()}, {members} ' f'FROM {self.t(TARGETS)} AS t ' f'JOIN {self.t(PROJECTS)} AS p ON p.tenant = t.tenant AND p.id = t.project_id ' - f'WHERE t.tenant = %s AND {self._listed_projection()} AND ({match}) ' - 'ORDER BY p.created_at, p.id', + f'WHERE t.tenant = %s AND ({match}) ORDER BY p.created_at, p.id', (self.tenant_slug, *(v for pair in part for v in pair))).fetchall() for row in rows: out[(row[0], row[1])].append(_record(row[2:])) diff --git a/platform/core/amazon_rivals.py b/platform/core/amazon_rivals.py index a42821c5f1c0d7d29b3c76f1f3ff2ec2a92783b6..b426ba1486fd647d318a1f0826428465fd75d08e 100644 --- a/platform/core/amazon_rivals.py +++ b/platform/core/amazon_rivals.py @@ -1,11 +1,10 @@ -"""Per-tenant PostgreSQL money-search rival pools and daily quotes. +"""Per-tenant PostgreSQL rivals of our ASINs, found from Brand Analytics, plus their daily quotes. Discovery reads the Amazon mirror through a cursor the caller opens: core imports no harness module (verify_layering) and no App module (rule 13). Tables are created on the first write. """ from contextlib import contextmanager -from dataclasses import dataclass from datetime import date, datetime, timedelta, timezone import json import re @@ -20,6 +19,8 @@ RUNS = 'amazon_rival_runs' WANTED = 'amazon_rival_wanted' TABLES = (RIVALS, DISCOVERY, QUOTES, RUNS, WANTED) +#: R23: at most this many auto-found rivals per ASIN, by click share. +AUTO_LIMIT = 10 #: Brand Analytics keeps the top 3 clicked ASINs of each search term. TOP_CLICKED = 3 SOURCES = frozenset(('auto', 'added')) @@ -28,35 +29,16 @@ CHUNK = 200 BA_TABLE = 'amzn_brand_analytics' LISTINGS_TABLE = 'amzn_listings' PRICING_TABLE = 'amzn_pricing' -SQP_TABLE = 'amzn_search_query_performance' -RANKS_TABLE = 'amzn_apify_rank_observations' -SNAPSHOTS_TABLE = 'amzn_competitor_snapshots' -IMAGES_TABLE = 'amzn_competitor_images' -POOL_SNAPSHOT_FIELDS = ('snapshot_date', 'brand', 'title', 'bullets', 'price', 'currency', 'prime', 'offers') -POOL_IMAGE_FIELDS = ('variant', 'link', 'width', 'height') NO_TOP3 = "Not in any search term's top 3 clicked" NO_WEEK = 'Brand Analytics has no dated weekly reports' -NO_MONTH = 'No completed monthly Search Query Performance report for this product yet.' -NO_MONEY = 'No searches with market money for this product yet.' -NO_KEYWORD_RANKS = "No keyword ranks for this product's searches yet." -NO_PRODUCT_TYPE = 'Product type not read yet. The next daily run reads it.' -WAITING_PRODUCT_TYPE = '{count} found, waiting for the daily read of their product type.' NOT_OURS = 'Not an ASIN in your Amazon listings' PARENT_ROLLS_UP = 'A Parent ASIN has no rival list. Its fields roll up its child ASINs.' -NO_POOL = 'No search report or keyword ranks for this product yet.' ASIN_RE = re.compile(r'[A-Z0-9]{10}') -POOL_FIELDS = { - RIVALS: {'origin': ('TEXT', "'click'"), 'money_won': ('DOUBLE PRECISION', '0'), - 'terms_won': ('INTEGER', '0'), 'term_rank': ('INTEGER', 'NULL'), - 'eligible': ('BOOLEAN', 'TRUE')}, - DISCOVERY: {'month': ('DATE', 'NULL'), 'keyword_ranks_on': ('DATE', 'NULL'), - 'product_type': ('TEXT', 'NULL'), 'held': ('INTEGER', '0')}, -} RIVAL_COLUMNS = ('asin', 'rival_asin', 'source', 'removed', 'click_share', 'term', 'title', 'week', - 'added_by', 'updated_at', *POOL_FIELDS[RIVALS]) -DISCOVERY_COLUMNS = ('asin', 'week', 'discovered_at', 'cause', 'auto_count', *POOL_FIELDS[DISCOVERY]) + 'added_by', 'updated_at') +DISCOVERY_COLUMNS = ('asin', 'week', 'discovered_at', 'cause', 'auto_count') QUOTE_COLUMNS = ('asin', 'quote_on', 'offer_price', 'buy_box_price', 'sales_rank', 'display_group', 'offer_cause', 'buy_box_cause', 'rank_cause', 'offers', 'currency', 'run_id', 'quoted_at') RUN_COLUMNS = ('id', 'started_at', 'progress', 'progress_at', 'finished_at', 'asin_total', 'priced', @@ -193,355 +175,96 @@ def children(cur, parent): return [r[0] for r in rows if r[0]] -def _money_sql(cur): - """Newest completed monthly searches per ASIN, including zero-money searches for freshness.""" - required = {'asin', 'search_query_data_search_query', 'start_date', 'end_date', - 'purchase_data_total_purchase_count', 'purchase_data_total_median_purchase_price_amount'} - if not required <= _columns(cur, SQP_TABLE): - return ('SELECT NULL::VARCHAR AS asin, NULL::VARCHAR AS term, NULL::DATE AS month, ' - 'NULL::DOUBLE AS market_money WHERE FALSE') - return f''' - WITH reports AS ( - SELECT upper(trim(CAST(asin AS VARCHAR))) AS asin, - lower(trim(CAST(search_query_data_search_query AS VARCHAR))) AS term, - TRY_CAST(start_date AS DATE) AS start_on, TRY_CAST(end_date AS DATE) AS end_on, - TRY_CAST(purchase_data_total_purchase_count AS DOUBLE) * - TRY_CAST(purchase_data_total_median_purchase_price_amount AS DOUBLE) AS market_money - FROM {SQP_TABLE} - WHERE date_diff('day', TRY_CAST(start_date AS DATE), TRY_CAST(end_date AS DATE)) > 7 - AND TRY_CAST(end_date AS DATE) < CURRENT_DATE), - latest AS ( - SELECT * FROM reports QUALIFY end_on = max(end_on) OVER (PARTITION BY asin)) - SELECT asin, term, CAST(date_trunc('month', start_on) AS DATE) AS month, - max(market_money) AS market_money - FROM latest GROUP BY asin, term, month''' - - -def _click_sql(cur): - """Normalized dated top-three clicks, with percentage shares kept in their source units.""" +def _candidates(cur, targets, week, per_asin): + """Ranked co-clicked ASINs per target, bounded in SQL to `per_asin` rows each.""" cols = _columns(cur, BA_TABLE) - if not {'search_term', 'clicked_asin', 'click_share_rank', 'click_share', - '_period_start', '_period_end'} <= cols: - return ('SELECT NULL::VARCHAR AS term, NULL::VARCHAR AS rival, NULL::DOUBLE AS share, ' - 'NULL::VARCHAR AS title, NULL::INTEGER AS term_rank, NULL::DATE AS week WHERE FALSE') - title = 'CAST(clicked_item_name AS VARCHAR)' if 'clicked_item_name' in cols else 'NULL::VARCHAR' - return f'''SELECT lower(trim(CAST(search_term AS VARCHAR))) AS term, - upper(trim(CAST(clicked_asin AS VARCHAR))) AS rival, - TRY_CAST(click_share AS DOUBLE) AS share, {title} AS title, - TRY_CAST(click_share_rank AS INTEGER) AS term_rank, {_week_sql()} AS week - FROM {BA_TABLE} WHERE {weekly_period_sql()} - AND TRY_CAST(click_share_rank AS INTEGER) BETWEEN 1 AND {TOP_CLICKED}''' - - -def _rank_rows_sql(cur): - """Raw Keyword ranks observations include rivals absent from the own-placement view.""" - if not {'keyword', 'asin', 'date', 'organic_rank', 'is_sponsored'} <= _columns(cur, RANKS_TABLE): - return ('SELECT NULL::VARCHAR AS term, NULL::VARCHAR AS rival, NULL::DATE AS observed_on, ' - 'NULL::INTEGER AS term_rank, NULL::BOOLEAN AS sponsored WHERE FALSE') - return f'''SELECT lower(trim(CAST(keyword AS VARCHAR))) AS term, - upper(trim(CAST(asin AS VARCHAR))) AS rival, TRY_CAST(date AS DATE) AS observed_on, - TRY_CAST(organic_rank AS INTEGER) AS term_rank, - TRY_CAST(is_sponsored AS BOOLEAN) AS sponsored FROM {RANKS_TABLE}''' - - -def _search_dates(cur, targets): - rows = _read(cur, f''' - WITH money AS ({_money_sql(cur)}), observations AS ({_rank_rows_sql(cur)}) - SELECT m.asin, max(m.month), max(o.observed_on), count(DISTINCT m.term) - FILTER (WHERE m.market_money > 0) - FROM money m LEFT JOIN observations o ON o.term = m.term AND m.market_money > 0 - WHERE m.asin IN {_in(targets)} GROUP BY m.asin''', targets) - return {a: dict(month=_day(m), keyword_ranks_on=_day(k), money_terms=n) for a, m, k, n in rows} - - -def _newest_snapshots_sql(cur, fields=('product_type', 'brand')): - if not {'asin', 'snapshot_date', 'product_type', 'brand'} <= _columns(cur, SNAPSHOTS_TABLE): - return 'SELECT NULL::VARCHAR AS asin, ' + ', '.join(f'NULL::VARCHAR AS {f}' for f in fields) + ' WHERE FALSE' - return (f'SELECT upper(trim(asin)) AS asin, {", ".join(fields)} FROM {SNAPSHOTS_TABLE} ' - 'QUALIFY row_number() OVER (PARTITION BY upper(trim(asin)) ORDER BY snapshot_date DESC) = 1') - - -def _brand(value): - return ''.join((value or '').casefold().split()) - - -def _same_type(target_type, rival_type): - return bool(target_type and rival_type and target_type == rival_type) - - -def _other_brand(brand, own_brands): - return not _brand(brand) or _brand(brand) not in own_brands - - -@dataclass -class PoolFacts: - """Newest ASIN facts and all historical own brands determine pool eligibility.""" - - latest: dict - own_brands: set - own_asins: set - - def product_type(self, target): - return self.latest.get(target, {}).get('product_type') - - def eligible(self, target, rival): - fact = self.latest.get(rival, {}) - return (rival not in self.own_asins and _same_type(self.product_type(target), fact.get('product_type')) - and _other_brand(fact.get('brand'), self.own_brands)) - - -def _pool_facts(cur, values, own, prefetched=None, candidate_rows=()): - latest = {r[1]: dict(product_type=r[-2], brand=r[-1]) for r in candidate_rows} - for part in _chunks(sorted(values - latest.keys())): - rows = _read(cur, f'SELECT asin, product_type, brand FROM ({_newest_snapshots_sql(cur)}) ' - f'WHERE asin IN {_in(part)}', part) - latest.update({a: dict(product_type=t, brand=b) for a, t, b in rows}) - brands = set() - if {'asin', 'brand'} <= _columns(cur, SNAPSHOTS_TABLE): - rows = _read(cur, f'SELECT DISTINCT s.brand FROM {SNAPSHOTS_TABLE} s ' - f'JOIN {LISTINGS_TABLE} l ON upper(trim(s.asin)) = upper(trim(l.asin1))') - brands.update(_brand(row[0]) for row in rows if _brand(row[0])) - latest.update(prefetched or {}) - brands.update(_brand(latest[a].get('brand')) for a in own if a in latest and _brand(latest[a].get('brand'))) - return PoolFacts(latest, brands, own) - - -def _candidates(cur, targets, week): - """Every non-own clicked or organic page-one ASIN on each target's monthly money searches.""" + share = 'TRY_CAST("click_share" AS DOUBLE)' if 'click_share' in cols else 'CAST(NULL AS DOUBLE)' + title = 'CAST("clicked_item_name" AS VARCHAR)' if 'clicked_item_name' in cols else 'CAST(NULL AS VARCHAR)' sql = f''' - WITH money AS ({_money_sql(cur)}), clicks AS ({_click_sql(cur)}), - observations AS ({_rank_rows_sql(cur)}), - newest_observations AS ( - SELECT * FROM observations - QUALIFY observed_on = max(observed_on) OVER (PARTITION BY term)), - click AS ( - SELECT term, rival, max(share) AS share, min(term_rank) AS term_rank, max(title) AS title - FROM clicks WHERE week = CAST(? AS DATE) GROUP BY term, rival), - page1 AS ( - SELECT term, rival, min(term_rank) AS term_rank FROM newest_observations - WHERE sponsored = FALSE GROUP BY term, rival), - placements AS ( - SELECT coalesce(c.term, p.term) AS term, coalesce(c.rival, p.rival) AS rival, - c.share, c.title, coalesce(c.term_rank, p.term_rank) AS term_rank, - c.term IS NOT NULL AS clicked, p.term IS NOT NULL AS page1 - FROM click c FULL OUTER JOIN page1 p ON p.term = c.term AND p.rival = c.rival), + WITH ours AS ( + SELECT DISTINCT upper(trim(CAST("asin1" AS VARCHAR))) AS asin FROM {LISTINGS_TABLE} + WHERE "asin1" IS NOT NULL), + week AS ( + SELECT "search_term" AS term, upper(trim(CAST("clicked_asin" AS VARCHAR))) AS clicked, + {share} AS share, {title} AS title + FROM {BA_TABLE} + WHERE TRY_CAST("click_share_rank" AS INTEGER) BETWEEN 1 AND {TOP_CLICKED} + AND ({weekly_period_sql()}) AND {_week_sql()} = CAST(? AS DATE)), hits AS ( - SELECT m.asin, p.*, m.market_money * coalesce(p.share, 0) / 100 AS money_won - FROM money m JOIN placements p ON p.term = m.term - WHERE m.asin IN {_in(targets)} AND m.market_money > 0 AND p.rival <> '' - AND p.rival NOT IN (SELECT upper(trim(CAST(asin1 AS VARCHAR))) - FROM {LISTINGS_TABLE} WHERE asin1 IS NOT NULL)), - totals AS ( - SELECT asin, rival, max(share) AS share, sum(money_won) AS money_won, - count(*) FILTER (WHERE clicked AND term_rank = 1) AS terms_won, - CASE WHEN bool_or(clicked) AND bool_or(page1) THEN 'both' - WHEN bool_or(clicked) THEN 'click' ELSE 'page1' END AS origin - FROM hits GROUP BY asin, rival), - representative AS ( - SELECT asin, rival, term, title, term_rank FROM hits - QUALIFY row_number() OVER (PARTITION BY asin, rival - ORDER BY money_won DESC, term_rank, term) = 1) - SELECT t.asin, t.rival, t.share, r.term, r.title, t.origin, t.money_won, t.terms_won, r.term_rank, - s.product_type, s.brand - FROM totals t JOIN representative r USING (asin, rival) - LEFT JOIN ({_newest_snapshots_sql(cur)}) s ON s.asin = t.rival - ORDER BY t.asin, t.money_won DESC, t.rival''' - return _read(cur, sql, (week, *targets)) + SELECT DISTINCT term, clicked AS asin FROM week WHERE clicked IN {_in(targets)}), + cands AS ( + SELECT h.asin, w.clicked AS rival, w.share, w.term, w.title + FROM hits h JOIN week w ON w.term = h.term + WHERE w.clicked <> '' AND w.clicked NOT IN (SELECT asin FROM ours)), + best AS ( + SELECT asin, rival, max(share) AS share, + arg_max(term, coalesce(share, -1)) AS term, arg_max(title, coalesce(share, -1)) AS title + FROM cands GROUP BY asin, rival), + ranked AS ( + SELECT *, row_number() OVER (PARTITION BY asin ORDER BY share DESC NULLS LAST, rival) AS rn + FROM best) + SELECT asin, rival, share, term, title FROM ranked WHERE rn <= ? ORDER BY asin, rn''' + return _read(cur, sql, (week, *targets, per_asin)) # --------------------------------------------------------------------------- discovery -def discover(rt, cur, asins=None, *, candidate_rows=None, prefetched=None): - """Refresh uncapped money-search pools while preserving removed and hand-added rivals.""" +def discover(rt, cur, asins=None): + """Rewrite the auto rivals of our ASINs (or of `asins`) from the newest dated week. + + A removed rival stays removed and a hand-added one stays added. Returns counts and `rowsRead`.""" backend = _backend(rt) requested = None if asins is None else _asins(asins) targets = sorted(our_asins(cur, requested)) stamp = _now() discovery = {} for missing in sorted(set(requested or ()) - set(targets)): - discovery[missing] = dict(asin=missing, week=None, discovered_at=stamp, cause=NOT_OURS, - auto_count=0, month=None, keyword_ranks_on=None, product_type=None, held=0) + discovery[missing] = dict(asin=missing, week=None, discovered_at=stamp, cause=NOT_OURS, auto_count=0) week = newest_week(cur) if targets else None - dates = _search_dates(cur, targets) if targets else {} - rows = (_candidates(cur, targets, week) if candidate_rows is None else candidate_rows) if targets else [] - stored = backend.pool_rows(targets) - before = backend.rivals_of(targets) - values = set(targets) | {r[1] for r in rows} | {r['rival_asin'] for r in stored} - facts = _pool_facts(cur, values, our_asins(cur), prefetched, rows) - found, held = _eligible_candidates(rows, stored, facts, week, stamp) + found = {} + rows_read = 0 + if targets and week is not None: + kept = backend.kept(targets) + per_asin = AUTO_LIMIT + max((len(v) for v in kept.values()), default=0) + rows = _candidates(cur, targets, week, per_asin) + rows_read = len(rows) + for target, rival, share, term, title in rows: + if rival in kept.get(target, ()) or len(found.get(target, ())) >= AUTO_LIMIT: + continue + found.setdefault(target, []).append(dict( + asin=target, rival_asin=rival, source='auto', removed=False, + click_share=None if share is None else float(share), term=term, title=title, + week=week, added_by=None, updated_at=stamp)) for target in targets: - count = (len(found.get(target, ())) if facts.product_type(target) else - sum(r['source'] == 'auto' for r in stored if r['asin'] == target and not r['removed'])) - meta = dates.get(target) or dict(month=None, keyword_ranks_on=None, money_terms=0) - cause = _pool_cause(facts.product_type(target), held.get(target, 0), week, meta, count) - discovery[target] = dict(asin=target, week=week, discovered_at=stamp, cause=cause, auto_count=count, - month=meta['month'], keyword_ranks_on=meta['keyword_ranks_on'], - product_type=facts.product_type(target), held=held.get(target, 0)) - ready = [t for t in targets if facts.product_type(t)] - backend.replace_auto(ready, [row for t in ready for row in found.get(t, ())], list(discovery.values())) - backend.set_eligibility(stored, facts) - members = backend.rivals_of(targets) - pools = {target: len(members.get(target, ())) for target in targets} + count = len(found.get(target, ())) + cause = NO_WEEK if week is None else (None if count else NO_TOP3) + discovery[target] = dict(asin=target, week=week, discovered_at=stamp, cause=cause, auto_count=count) + backend.replace_auto(targets, [row for t in targets for row in found.get(t, ())], list(discovery.values())) return {'asins': len(targets), 'rivals': sum(len(v) for v in found.values()), 'week': week, - 'pools': pools, 'rowsRead': len(rows), 'held': held, - 'changed': tuple(t for t in targets if set(before.get(t, ())) != set(members.get(t, ()))), - 'causes': {k: v['cause'] for k, v in discovery.items() if v['cause']}} - - -def _eligible_candidates(rows, stored, facts, week, stamp): - kept = {(r['asin'], r['rival_asin']) for r in stored if r['removed'] or r['source'] == 'added'} - candidates = {(r[0], r[1]) for r in rows} | { - (r['asin'], r['rival_asin']) for r in stored if r['source'] == 'added' and not r['removed']} - removed = {(r['asin'], r['rival_asin']) for r in stored if r['removed']} - held = {} - for target, rival in candidates - removed: - if (rival not in facts.own_asins and not facts.product_type(rival) - and _other_brand(facts.latest.get(rival, {}).get('brand'), facts.own_brands)): - held[target] = held.get(target, 0) + 1 - found = {} - for target, rival, share, term, title, origin, money, terms, term_rank, _type, _name in rows: - if (target, rival) in kept or not facts.eligible(target, rival): - continue - found.setdefault(target, []).append(dict( - asin=target, rival_asin=rival, source='auto', removed=False, eligible=True, - click_share=None if share is None else float(share), term=term, title=title, - week=week, added_by=None, updated_at=stamp, origin=origin, - money_won=float(money), terms_won=int(terms), term_rank=term_rank)) - return found, held - - -def _discovery_cause(week, meta, count): - if meta['month'] is None: - return NO_MONTH - if not meta['money_terms']: - return NO_MONEY - if meta['keyword_ranks_on'] is None: - return NO_KEYWORD_RANKS - if week is None: - return NO_WEEK - return None if count else NO_TOP3 - - -def _pool_cause(product_type, held, week, meta, count): - if not product_type: - return NO_PRODUCT_TYPE - if held: - return WAITING_PRODUCT_TYPE.format(count=held) - return _discovery_cause(week, meta, count) + 'rowsRead': rows_read, 'causes': {k: v['cause'] for k, v in discovery.items() if v['cause']}} def stale(rt, cur, target): - """True when the ASIN's discovery predates a weekly, monthly or Keyword ranks source.""" + """True when this ASIN's discovery is missing or predates the newest dated week.""" row = _backend(rt).discovery([target]).get(target) if row is None: return True - types = _read(cur, f'SELECT product_type FROM ({_newest_snapshots_sql(cur)}) WHERE asin = ?', [target]) - product_type = types[0][0] if types else None - if row['product_type'] != product_type or (not product_type and row['cause'] != NO_PRODUCT_TYPE): - return True - newest = {'week': newest_week(cur), **(_search_dates(cur, [target]).get(target) or {})} - return any(newest.get(key) is not None and - (row.get(key) is None or row[key] < newest[key]) - for key in ('week', 'month', 'keyword_ranks_on')) + newest = newest_week(cur) + return newest is not None and (row['week'] is None or row['week'] < newest) # --------------------------------------------------------------------------- doors def rivals(rt, target): - """One ASIN's rivals: hand-added first, then money won and rank average, with discovery dates.""" + """One ASIN's rivals: hand-added first, then auto by click share, with its discovery row.""" target = asin(target) backend = _backend(rt) rows = backend.rivals(target) found = backend.discovery([target]).get(target) return {'asin': target, 'rivals': rows, 'removed': backend.removed_count(target), 'week': found['week'] if found else None, - 'month': found['month'] if found else None, - 'keywordRanksOn': found['keyword_ranks_on'] if found else None, 'discoveredAt': found['discovered_at'] if found else None, - 'cause': found['cause'] if found else None, - 'productType': found['product_type'] if found else None, - 'held': found['held'] if found else 0} - - -def pool(rt, cur, target): - """Read eligible pool rows through the caller's mirror cursor without writes or provider reads.""" - target = asin(target) - backend = _backend(rt) - if cur is None: - return {**rivals(rt, target), 'rivals': [], 'competitors': [], 'dataAsOf': None, 'cause': NO_POOL} - own = our_asins(cur) - week = newest_week(cur) - sources = _candidates(cur, [target], week) if target in own else [] - stored = backend.pool_rows([target]) - facts = _pool_facts(cur, {target} | {r[1] for r in sources} | {r['rival_asin'] for r in stored}, - own, candidate_rows=sources) - found, held = _eligible_candidates(sources, stored, facts, week, _now()) - if facts.product_type(target): - rows = found.get(target, []) + [dict(r, eligible=True) for r in stored - if r['source'] == 'added' and not r['removed'] - and facts.eligible(target, r['rival_asin'])] - else: - rows = [r for r in stored if r['eligible'] and not r['removed'] and r['rival_asin'] not in own - and _other_brand(facts.latest.get(r['rival_asin'], {}).get('brand'), facts.own_brands)] - if target not in own: - rows = [] - averages = backend.rank_averages([r['rival_asin'] for r in rows]) - rows = [{**r, **averages.get(r['rival_asin'], {'rank_average': None, 'rank_days': 0})} for r in rows] - meta = _search_dates(cur, [target]).get(target) or dict(month=None, keyword_ranks_on=None, money_terms=0) - cause = _pool_cause(facts.product_type(target), held.get(target, 0), week, meta, len(rows)) - rows = sorted(rows, key=_pool_order) - snapshots = _pool_snapshots(cur, [target, *(r['rival_asin'] for r in rows)]) - images = _pool_images(cur, [r['rival_asin'] for r in rows]) - dates = [s['snapshot_date'] for s in snapshots.values() if s['snapshot_date'] is not None] - return {**rivals(rt, target), 'rivals': rows, 'week': week, - 'month': meta['month'], 'keywordRanksOn': meta['keyword_ranks_on'], - 'productType': facts.product_type(target), 'held': held.get(target, 0), - 'cause': cause if target in own else NOT_OURS, 'dataAsOf': max(dates, default=None), - 'competitors': [_pool_competitor(r, snapshots.get(r['rival_asin'], {}), - images.get(r['rival_asin'], [])) for r in rows]} - - -def _pool_snapshots(cur, values): - snapshots = {} - for part in _chunks(sorted(set(values))): - rows = _read(cur, f'SELECT * FROM ({_newest_snapshots_sql(cur, POOL_SNAPSHOT_FIELDS)}) ' - f'WHERE asin IN {_in(part)}', part) - for target, *fields in rows: - snapshot = dict(zip(POOL_SNAPSHOT_FIELDS, fields)) - snapshot['snapshot_date'] = _day(snapshot['snapshot_date']) - snapshot['bullets'] = json.loads(snapshot['bullets'] or '[]') - snapshots[target] = snapshot - return snapshots - - -def _pool_images(cur, values): - """Resolve only the newest snapshot's variants against changed-link history.""" - images = {} - if not {'asin', *POOL_IMAGE_FIELDS} <= _columns(cur, IMAGES_TABLE): - return images - for part in _chunks(values): - rows = _read(cur, f''' - SELECT s.asin, i.variant, i.link, i.width, i.height - FROM ({_newest_snapshots_sql(cur, ('image_variants',))}) s, - json_each(s.image_variants) v, {IMAGES_TABLE} i - WHERE s.asin IN {_in(part)} AND i.asin = s.asin - AND i.variant = json_extract_string(v.value, '$.variant') - AND i.link = json_extract_string(v.value, '$.link') - ORDER BY s.asin, CAST(v.key AS INTEGER)''', part) - for target, *fields in rows: - images.setdefault(target, []).append(dict(zip(POOL_IMAGE_FIELDS, fields))) - return images - - -def _pool_competitor(row, snapshot, images): - days = row['rank_days'] - return {'asin': row['rival_asin'], 'brand': snapshot.get('brand'), 'title': snapshot.get('title'), - 'origin': row['origin'], 'moneyWon': row['money_won'], 'termsWon': row['terms_won'], - 'term': row['term'], 'termRank': row['term_rank'], 'bestSellerRank': row['rank_average'], - 'rankDays': days, 'rankCause': f'Rank average needs 7 days of ranks. {days} so far.' if days < 7 else None, - 'price': snapshot.get('price'), 'currency': snapshot.get('currency'), 'prime': snapshot.get('prime'), - 'offers': snapshot.get('offers'), 'imageCount': len(images), 'images': images, - 'bullets': snapshot.get('bullets', []), 'snapshotOn': snapshot.get('snapshot_date')} + 'cause': found['cause'] if found else None} def add(rt, target, rival, *, user, title=None): @@ -549,10 +272,9 @@ def add(rt, target, rival, *, user, title=None): target, rival = asin(target), asin(rival, 'rival') if rival == target: raise ValueError('An ASIN cannot be its own rival.') - return _backend(rt).add(dict(asin=target, rival_asin=rival, source='added', removed=False, eligible=False, + return _backend(rt).add(dict(asin=target, rival_asin=rival, source='added', removed=False, click_share=None, term=None, title=title, week=None, - added_by=str(user or '') or None, updated_at=_now(), - origin='added', money_won=0, terms_won=0, term_rank=None)) + added_by=str(user or '') or None, updated_at=_now())) def remove(rt, target, rival, *, user): @@ -616,19 +338,6 @@ def runs(rt, limit=10): return _backend(rt).runs(int(limit)) -def rank_average(rt, asins): - """Mean daily rank in each ASIN's newest display group, requiring seven of its newest 30 days.""" - values = _asins(asins) - found = _backend(rt).rank_averages(values) if values else {} - return {a: (found.get(a) or {}).get('rank_average') for a in values} - - -def _pool_order(row): - average = row.get('rank_average') - return (-float(row.get('money_won') or 0), - average is None, average if average is not None else 0, row['rival_asin']) - - # --------------------------------------------------------------------------- the daily quotes run #: The run row gets a progress marker after each block of this many ASINs. @@ -642,8 +351,7 @@ MAX_BACKOFF_SECONDS = 300.0 #: An unfinished run, or a finished run with a cause, is retried after this many hours. RETRY_RUN_HOURS = 3 -READS = {'offers': 'offers', 'summaries': 'Buy Box price', 'facts': 'catalog facts'} -FACT_COUNT_COLUMNS = {'title': 'title', 'product_type': 'product_type', 'image': 'image', 'rank': 'sales_rank'} +READS = {'offers': 'offers', 'summaries': 'Buy Box price', 'ranks': 'sales rank'} NO_MIRROR = 'The Amazon data is not on this worker yet, so no ASIN was quoted.' NO_OWN_OFFER = 'Amazon returned no offer of yours on this ASIN.' NO_BUY_BOX = 'Amazon returned no Buy Box offer for this ASIN.' @@ -741,7 +449,7 @@ def _summary_quote(response): def _keyed(kind, got): - if kind == 'facts': + if kind == 'ranks': return {str(k).upper(): v for k, v in (got or {}).items()} if kind == 'offers': return {str((((r or {}).get('body') or {}).get('payload') or {}).get('ASIN') or '').upper(): r @@ -781,16 +489,7 @@ def _read_batches(kind, fn, asins, *, sleep, log, counts): attempt = 0 while True: try: - got = _keyed(kind, fn(part)) - answers.update(got) - if kind == 'facts': - for target, fact in got.items(): - if fact.get('cause'): - causes[target] = fact['cause'] - if '(429)' in fact['cause']: - counts['throttled'] += 1 - elif 'Amazon refused' in fact['cause']: - counts['refused'] += 1 + answers.update(_keyed(kind, fn(part))) break except Exception as exc: # noqa: BLE001 if _throttled(exc) and attempt < THROTTLE_RETRIES: @@ -806,19 +505,14 @@ def _read_batches(kind, fn, asins, *, sleep, log, counts): return answers, causes -def _quote_block(block, own, readers, seller_id, quote_on, run_id, sleep, log, counts, - prefetched=None, fact_causes=None): +def _quote_block(block, own, readers, seller_id, quote_on, run_id, sleep, log, counts): """One quote row per ASIN of `block`, each with a value or a cause for every field it owns.""" mine = [a for a in block if a in own] others = [a for a in block if a not in own] offered, offer_causes = _read_batches('offers', readers['offers'], mine, sleep=sleep, log=log, counts=counts) summed, sum_causes = _read_batches('summaries', readers['summaries'], others, sleep=sleep, log=log, counts=counts) - cached = prefetched or {} - facts, rank_causes = _read_batches('facts', readers['facts'], [a for a in block if a not in cached], - sleep=sleep, log=log, counts=counts) - facts.update({a: cached[a] for a in block if a in cached}) - rank_causes.update(fact_causes or {}) + ranked, rank_causes = _read_batches('ranks', readers['ranks'], block, sleep=sleep, log=log, counts=counts) stamp = _now() rows = [] for target in block: @@ -837,31 +531,20 @@ def _quote_block(block, own, readers, seller_id, quote_on, run_id, sleep, log, c row.update(_summary_quote(summed[target])) else: row['buy_box_cause'] = NO_PRICE - rank = facts.get(target) or {} + rank = ranked.get(target) or {} value = rank.get('rank') if isinstance(rank.get('rank'), int) and not isinstance(rank.get('rank'), bool) else None row.update(sales_rank=value, display_group=rank.get('group') if value is not None else None, - rank_cause=None if value is not None else rank_causes.get(target) or NO_RANK) + rank_cause=rank_causes.get(target) or (None if value is not None else NO_RANK)) rows.append(row) - return rows, [_snapshot(row, facts.get(row['asin']) or {}, row['asin'] in own) for row in rows] - - -def _snapshot(quote, facts, own): - """C3 facts with price and rank taken from the exact persisted quote row.""" - return {**{key: facts.get(key) for key in ('product_type', 'brand', 'title', 'bullets', - 'image_variants', 'prime', 'offers', 'rank_cat', 'image')}, - 'asin': quote['asin'], 'snapshot_date': quote['quote_on'], - 'price': quote['offer_price' if own else 'buy_box_price'], 'currency': quote['currency'], - 'sales_rank': quote['sales_rank'], 'display_group': quote['display_group']} + return rows -def _tally(counts, rows, own, snapshots): +def _tally(counts, rows, own): for row in rows: counts['priced'] += int((row['offer_price'] if row['asin'] in own else None) is not None or row['buy_box_price'] is not None) counts['ranked'] += int(row['sales_rank'] is not None) counts['caused'] += int(bool(row['offer_cause'] or row['buy_box_cause'] or row['rank_cause'])) - for key, field in FACT_COUNT_COLUMNS.items(): - counts['facts'][key] += sum(row.get(field) is not None and row[field] != '' for row in snapshots) def _changed(before, row): @@ -869,47 +552,35 @@ def _changed(before, row): return old is None or any(old.get(k) != row.get(k) for k in QUOTED_FIELDS) -def _prepare_pools(rt, open_mirror, facts_reader, sleep, log, counts): - with open_mirror() as cur: - if cur is None: - return None - own = our_asins(cur) - sources = _candidates(cur, sorted(own), newest_week(cur)) if own else [] - stored = _backend(rt).pool_rows(sorted(own)) - values = own | {r[1] for r in sources} | {r['rival_asin'] for r in stored if not r['removed']} - known = _pool_facts(cur, values, own, candidate_rows=sources) - missing = sorted(a for a in values if not known.product_type(a)) - prefetched, causes = _read_batches('facts', facts_reader, missing, sleep=sleep, log=log, counts=counts) - prefetched = {a: prefetched.get(a, {}) for a in missing} - with open_mirror() as cur: - if cur is None: - return None - found = discover(rt, cur, candidate_rows=sources, prefetched=prefetched) - return own, found, prefetched, causes - - -def run_quotes(rt, open_mirror, *, offers, summaries, facts, seller_id=None, run_id=None, now=None, +def run_quotes(rt, open_mirror, *, offers, summaries, ranks, seller_id=None, run_id=None, now=None, log=None, sleep=time.sleep): - """Discover, read unknown types outside the mirror, filter pools, then quote with cached facts.""" + """Rediscover rivals, then quote our ASINs, every rival and every wanted ASIN for today. + + `open_mirror()` is a context manager yielding a mirror cursor or None. It closes before the first + SP-API read. `offers`, `summaries` and `ranks` each take a list of at most 20 ASINs and return + `item_offers` responses, `competitive_summaries` responses and `{asin: {rank, group}}`. A raised + read is a cause on each ASIN of its batch, and a 429 backs off first. Returns the run's counts + and `changed`, the ASINs whose quote differs from their previous newest quote.""" _backend(rt) log = log or (lambda _message: None) now = now or datetime.now(timezone.utc) quote_on = now.date().isoformat() run_id = str(run_id or f'{quote_on}-{uuid.uuid4().hex[:8]}') counts = dict(ours=0, rivals=0, wanted=0, asins=0, priced=0, ranked=0, caused=0, refused=0, throttled=0, - backoffs=0, discovered=0, week=None, pools={}, facts=dict.fromkeys(FACT_COUNT_COLUMNS, 0)) + backoffs=0, discovered=0, week=None) start_run(rt, run_id, asin_total=0) log(f'run {run_id} started') done = 0 try: try: - prepared = _prepare_pools(rt, open_mirror, facts, sleep, log, counts) - if prepared is None: - return _finish(rt, run_id, done, counts, NO_MIRROR, log) - own, found, prefetched, fact_causes = prepared + with open_mirror() as cur: + if cur is None: + return _finish(rt, run_id, done, counts, NO_MIRROR, log) + found = discover(rt, cur) + own = our_asins(cur) except MirrorUnreadable as exc: return _finish(rt, run_id, done, counts, f'The Amazon data could not be read ({exc}).', log) - counts.update(discovered=found['rivals'], week=found['week'], pools=found['pools']) + counts.update(discovered=found['rivals'], week=found['week']) others = sorted(set(rival_asins(rt)) - own) extra = sorted(set(wanted(rt)) - own - set(others)) targets = sorted(own) + others + extra @@ -918,23 +589,18 @@ def run_quotes(rt, open_mirror, *, offers, summaries, facts, seller_id=None, run log(f'run {run_id} quotes {len(targets)} ASINs: {len(own)} ours, {len(others)} rivals, ' f'{len(extra)} wanted') before = newest_quotes(rt, targets) if targets else {} - readers = {'offers': offers, 'summaries': summaries, 'facts': facts} - changed, snapshots = list(found['changed']), [] + readers = {'offers': offers, 'summaries': summaries, 'ranks': ranks} + changed = [] for block in _chunks(targets, PROGRESS_EVERY): - rows, captured = _quote_block(block, own, readers, seller_id, quote_on, run_id, sleep, log, counts, - prefetched, fact_causes) + rows = _quote_block(block, own, readers, seller_id, quote_on, run_id, sleep, log, counts) put_quotes(rt, rows) - snapshots.extend(captured) - _tally(counts, rows, own, captured) + _tally(counts, rows, own) changed += [row['asin'] for row in rows if _changed(before, row)] done += len(block) mark_run(rt, run_id, progress=done, counts=counts) log(f'run {run_id} progress {done}/{len(targets)}') - cause = (f'Amazon could not finish every read at its rate or refused a read. ' - f'Retry the run within {RETRY_RUN_HOURS} hours after the rate limit delay.' - if counts['throttled'] or counts['refused'] else None) - result = _finish(rt, run_id, done, counts, cause, log) - result.update(changed=tuple(dict.fromkeys(changed)), snapshots=snapshots) + result = _finish(rt, run_id, done, counts, None, log) + result['changed'] = tuple(changed) return result except Exception as exc: mark_run(rt, run_id, progress=done, finished=True, counts=counts, @@ -947,7 +613,7 @@ def _finish(rt, run_id, done, counts, cause, log): mark_run(rt, run_id, progress=done, finished=True, counts=counts, cause=cause) log(f'run {run_id} finished: {counts["asins"]} ASINs, {counts["priced"]} priced, {counts["ranked"]} ranked, ' f'{counts["caused"]} with a cause' + (f'. {cause}' if cause else '')) - return {'id': run_id, 'progress': done, 'cause': cause, 'counts': dict(counts), 'changed': (), 'snapshots': []} + return {'id': run_id, 'progress': done, 'cause': cause, 'counts': dict(counts), 'changed': ()} # --------------------------------------------------------------------------- competition status and rollups @@ -1217,70 +883,6 @@ def validate(rt, tables, plan): 'rows': len(parents), 'mismatches': bad['parent']}] -# --------------------------------------------------------------------------- pool validation - -def _pool_expected(cur, targets): - """Recompute raw click money independently of discovery's candidate and money queries.""" - required = {'asin', 'search_query_data_search_query', 'start_date', 'end_date', - 'purchase_data_total_purchase_count', 'purchase_data_total_median_purchase_price_amount'} - clicks = {'search_term', 'clicked_asin', 'click_share', 'click_share_rank', '_period_start', '_period_end'} - if not required <= _columns(cur, SQP_TABLE) or not clicks <= _columns(cur, BA_TABLE): - return {} - rows = _read(cur, f''' - WITH monthly AS ( - SELECT upper(trim(CAST(asin AS VARCHAR))) AS owner, - lower(trim(CAST(search_query_data_search_query AS VARCHAR))) AS query, - TRY_CAST(end_date AS DATE) AS ends, - TRY_CAST(purchase_data_total_purchase_count AS DOUBLE) * - TRY_CAST(purchase_data_total_median_purchase_price_amount AS DOUBLE) AS money - FROM {SQP_TABLE} - WHERE TRY_CAST(end_date AS DATE) > TRY_CAST(start_date AS DATE) + INTERVAL '7 days' - AND TRY_CAST(end_date AS DATE) < CURRENT_DATE), - newest_month AS (SELECT owner, max(ends) AS ends FROM monthly GROUP BY owner), - searches AS ( - SELECT m.owner, m.query, max(m.money) AS money FROM monthly m - JOIN newest_month n ON n.owner = m.owner AND n.ends = m.ends - GROUP BY m.owner, m.query HAVING max(m.money) > 0), - report AS ( - SELECT lower(trim(CAST(search_term AS VARCHAR))) AS query, - upper(trim(CAST(clicked_asin AS VARCHAR))) AS rival, - TRY_CAST(click_share AS DOUBLE) AS share, - TRY_CAST(click_share_rank AS INTEGER) AS position, {_week_sql()} AS starts - FROM {BA_TABLE} WHERE {weekly_period_sql()}), - term_clicks AS ( - SELECT query, rival, max(share) AS share, min(position) AS position FROM report - WHERE starts = (SELECT max(starts) FROM report) AND position BETWEEN 1 AND 3 - GROUP BY query, rival) - SELECT s.owner, c.rival, sum(s.money * coalesce(c.share, 0) * 0.01), - sum(CASE WHEN c.position = 1 THEN 1 ELSE 0 END) - FROM searches s JOIN term_clicks c ON c.query = s.query - WHERE s.owner IN {_in(targets)} GROUP BY s.owner, c.rival''', targets) - return {(a, r): (float(m), int(n)) for a, r, m, n in rows} - - -def validate_pool(cur, rt, targets): - """Tie stored auto-pool money and term wins to an independent raw-source aggregation.""" - targets = _asins(targets) - expected = _pool_expected(cur, targets) if targets else {} - bad_money, bad_terms, count = [], [], 0 - backend = _backend(rt) - for target in targets: - for row in backend.rivals(target): - if row['source'] != 'auto': - continue - count += 1 - money, terms = expected.get((target, row['rival_asin']), (0, 0)) - label = f'{target} {row["rival_asin"]}' - if _differs(row['money_won'], money, 0.000001): - bad_money.append(label) - if row['terms_won'] != terms: - bad_terms.append(label) - return [{'check': 'pool money won ties to Brand Analytics and search report rows', - 'ok': not bad_money, 'rows': count, 'mismatches': bad_money}, - {'check': 'pool terms won ties to Brand Analytics and search report rows', - 'ok': not bad_terms, 'rows': count, 'mismatches': bad_terms}] - - # --------------------------------------------------------------------------- storage def _ddl(table): @@ -1351,10 +953,6 @@ class PgRivalStore: """Create every rival table and index, only on a write.""" for name in TABLES: con.execute(_ddl(self.t(name))[name]) - for name, fields in POOL_FIELDS.items(): - for column, (kind, default) in fields.items(): - con.execute(f'ALTER TABLE {self.t(name)} ADD COLUMN IF NOT EXISTS {column} {kind} ' - f'DEFAULT {default}') con.execute(f'CREATE INDEX IF NOT EXISTS "amazon_rivals_rival" ON {self.t(RIVALS)} (tenant, rival_asin)') @contextmanager @@ -1364,49 +962,29 @@ class PgRivalStore: self.ensure_schema(con) yield con - def _select(self, con, name, columns, where, args, order='', *, include_ineligible=False): + def _select(self, con, name, columns, where, args, order=''): if not self._exists(con, name): return [] - projection = columns - fields = POOL_FIELDS.get(name, {}) - if fields.keys() & set(columns) or name == RIVALS: - existing = {r[0] for r in con.execute( - 'SELECT column_name FROM information_schema.columns WHERE table_schema = %s AND table_name = %s', - (self.schema, name)).fetchall()} - projection = [f'{fields[c][1]} AS {c}' if c in fields and c not in existing else c for c in columns] - if name == RIVALS and 'eligible' in existing and not include_ineligible: - where = f'({where}) AND eligible = TRUE' if where else 'eligible = TRUE' - rows = con.execute(f'SELECT {", ".join(projection)} FROM {self.t(name)} WHERE tenant = %s' + rows = con.execute(f'SELECT {", ".join(columns)} FROM {self.t(name)} WHERE tenant = %s' f'{" AND " + where if where else ""}{order}', (self.tenant_slug, *args)).fetchall() return [dict(zip(columns, row)) for row in rows] @staticmethod def _rival(row): - return {**row, 'removed': bool(row['removed']), 'eligible': bool(row['eligible']), 'week': _day(row['week']), - 'origin': 'added' if row['source'] == 'added' else row['origin'], + return {**row, 'removed': bool(row['removed']), 'week': _day(row['week']), 'click_share': None if row['click_share'] is None else float(row['click_share'])} - def pool_rows(self, targets): - """Stored choices include hidden additions and removal tombstones.""" + def kept(self, targets): + """`{asin: {rival}}` of removed and hand-added rivals, which discovery must not touch.""" + out = {} with self._connection() as con: - return [self._rival(row) for part in _chunks(list(targets)) for row in self._select( - con, RIVALS, RIVAL_COLUMNS, f'asin IN ({_marks(len(part))})', part, - include_ineligible=True)] - - def set_eligibility(self, rows, facts): - with self._write() as con: - for row in rows: - if row['removed']: - continue - target, rival = row['asin'], row['rival_asin'] - allowed = (facts.eligible(target, rival) if facts.product_type(target) else - row['eligible'] and rival not in facts.own_asins - and _other_brand(facts.latest.get(rival, {}).get('brand'), facts.own_brands)) - if allowed != row['eligible']: - con.execute(f'UPDATE {self.t(RIVALS)} SET eligible = %s ' - 'WHERE tenant = %s AND asin = %s AND rival_asin = %s', - (allowed, self.tenant_slug, target, rival)) + for part in _chunks(targets): + for row in self._select(con, RIVALS, ('asin', 'rival_asin'), + f"asin IN ({_marks(len(part))}) AND (removed = %s OR source = 'added')", + (*part, True)): + out.setdefault(row['asin'], set()).add(row['rival_asin']) + return out def replace_auto(self, targets, rows, discovery): with self._write() as con: @@ -1425,9 +1003,7 @@ class PgRivalStore: + ', '.join(f'({_marks(len(cols))})' for _ in part) + ' ON CONFLICT (tenant, asin) DO UPDATE SET week = excluded.week, ' 'discovered_at = excluded.discovered_at, cause = excluded.cause, ' - 'auto_count = excluded.auto_count, month = excluded.month, ' - 'keyword_ranks_on = excluded.keyword_ranks_on, ' - 'product_type = excluded.product_type, held = excluded.held', + 'auto_count = excluded.auto_count', tuple(v for row in part for v in (self.tenant_slug, *(row[c] for c in DISCOVERY_COLUMNS)))) def discovery(self, targets): @@ -1435,41 +1011,14 @@ class PgRivalStore: rows = [] for part in _chunks(list(targets)): rows += self._select(con, DISCOVERY, DISCOVERY_COLUMNS, f'asin IN ({_marks(len(part))})', part) - return {row['asin']: {**row, **{key: _day(row[key]) for key in - ('week', 'month', 'keyword_ranks_on')}} for row in rows} + return {row['asin']: {**row, 'week': _day(row['week'])} for row in rows} def rivals(self, target): with self._connection() as con: - rows = self._select(con, RIVALS, RIVAL_COLUMNS, 'asin = %s AND removed = %s', (target, False)) - averages = self.rank_averages([row['rival_asin'] for row in rows]) - enriched = [{**self._rival(row), **averages.get(row['rival_asin'], - {'rank_average': None, 'rank_days': 0})} for row in rows] - return sorted(enriched, key=lambda row: (row['source'] != 'added', *_pool_order(row))) - - def rank_averages(self, values): - out = {} - with self._connection() as con: - if not self._exists(con, QUOTES): - return out - for part in _chunks(values): - rows = con.execute(f''' - WITH newest AS ( - SELECT asin, max(quote_on) AS newest FROM {self.t(QUOTES)} - WHERE tenant = %s AND asin IN ({_marks(len(part))}) GROUP BY asin), - groups AS ( - SELECT n.asin, n.newest, q.display_group FROM newest n - JOIN {self.t(QUOTES)} q ON q.asin = n.asin AND q.quote_on = n.newest - WHERE q.tenant = %s) - SELECT q.asin, CASE WHEN count(q.sales_rank) >= 7 THEN avg(q.sales_rank) END, - count(q.sales_rank) - FROM groups m JOIN {self.t(QUOTES)} q ON q.asin = m.asin - AND q.display_group = m.display_group AND q.quote_on >= m.newest - 29 - AND q.quote_on <= m.newest - WHERE q.tenant = %s GROUP BY q.asin''', - (self.tenant_slug, *part, self.tenant_slug, self.tenant_slug)).fetchall() - out.update({a: dict(rank_average=None if avg is None else float(avg), rank_days=n) - for a, avg, n in rows}) - return out + rows = self._select(con, RIVALS, RIVAL_COLUMNS, 'asin = %s AND removed = %s', (target, False), + " ORDER BY CASE WHEN source = 'added' THEN 0 ELSE 1 END, " + 'CASE WHEN click_share IS NULL THEN 1 ELSE 0 END, click_share DESC, rival_asin') + return [self._rival(row) for row in rows] def removed_count(self, target): with self._connection() as con: @@ -1483,15 +1032,11 @@ class PgRivalStore: with self._write() as con: got = con.execute(f'INSERT INTO {self.t(RIVALS)} ({", ".join(cols)}) VALUES ({_marks(len(cols))}) ' "ON CONFLICT (tenant, asin, rival_asin) DO UPDATE SET source = 'added', " - "origin = 'added', money_won = 0, terms_won = 0, term_rank = NULL, " 'removed = excluded.removed, added_by = excluded.added_by, ' 'updated_at = excluded.updated_at, ' 'title = coalesce(excluded.title, title) ' f'RETURNING {", ".join(RIVAL_COLUMNS)}', (self.tenant_slug, *(row[c] for c in RIVAL_COLUMNS))).fetchone() - if not dict(zip(RIVAL_COLUMNS, got))['eligible']: - con.execute(f'UPDATE {self.t(DISCOVERY)} SET week = NULL, month = NULL, keyword_ranks_on = NULL ' - 'WHERE tenant = %s AND asin = %s', (self.tenant_slug, row['asin'])) return self._rival(dict(zip(RIVAL_COLUMNS, got))) def remove(self, target, rival, user, stamp): @@ -1505,13 +1050,15 @@ class PgRivalStore: return self._rival(dict(zip(RIVAL_COLUMNS, got))) def rivals_of(self, targets): - """`{asin: [rival ASIN]}` of the rivals not removed.""" + """`{asin: [rival ASIN]}` of the rivals not removed, hand-added first then by click share.""" out = {} with self._connection() as con: for part in _chunks(list(targets)): for row in self._select(con, RIVALS, ('asin', 'rival_asin'), f'asin IN ({_marks(len(part))}) AND removed = %s', (*part, False), - ' ORDER BY asin, rival_asin'): + " ORDER BY asin, CASE WHEN source = 'added' THEN 0 ELSE 1 END, " + 'CASE WHEN click_share IS NULL THEN 1 ELSE 0 END, click_share DESC, ' + 'rival_asin'): out.setdefault(row['asin'], []).append(row['rival_asin']) return out @@ -1532,10 +1079,6 @@ class PgRivalStore: with self._connection() as con: if not (self._exists(con, QUOTES) and self._exists(con, RIVALS)): return own, rows - columns = {r[0] for r in con.execute( - 'SELECT column_name FROM information_schema.columns WHERE table_schema = %s AND table_name = %s', - (self.schema, RIVALS)).fetchall()} - eligibility = 'AND r.eligible = TRUE ' if 'eligible' in columns else '' for part in _chunks(list(targets)): marks = _marks(len(part)) for target, rank, group in con.execute( @@ -1546,8 +1089,7 @@ class PgRivalStore: rows += [tuple(r) for r in con.execute( f'SELECT r.asin, q.{RIVAL_PRICE}, q.sales_rank, q.display_group FROM {self.t(RIVALS)} AS r ' f'JOIN {self.t(QUOTES)} AS q ON q.tenant = r.tenant AND q.asin = r.rival_asin ' - f'WHERE r.tenant = %s AND r.removed = %s {eligibility}' - f'AND r.asin IN ({marks}) AND {newest}', + f'WHERE r.tenant = %s AND r.removed = %s AND r.asin IN ({marks}) AND {newest}', (self.tenant_slug, False, *part)).fetchall()] return own, rows diff --git a/platform/core/connector_spec.py b/platform/core/connector_spec.py index ff83ec8f0a986dd97a4693b679fab3312a3eaecf..e5c42473214ccac2f47db20f837e5e3e763d9a9e 100644 --- a/platform/core/connector_spec.py +++ b/platform/core/connector_spec.py @@ -843,16 +843,6 @@ AMAZON_TABLES = ( identity=("rank_id",), links=(("keyword_id", _AMZN + "sku_keywords", "keyword_id", "many_to_one"),), ), - ConnectorTableSpec( - key=_AMZN + "competitor_snapshots", native="competitor_run:amazon_quotes_loop.run_tenant", - source="amzn_competitor_snapshots", label="Amazon Marketing: Competitor snapshots", pinned="asin", - identity=("asin", "snapshot_date"), - ), - ConnectorTableSpec( - key=_AMZN + "competitor_images", native="competitor_run:amazon_quotes_loop.run_tenant", - source="amzn_competitor_images", label="Amazon Marketing: Competitor images", pinned="asin", - identity=("asin", "variant", "link"), - ), ) #: Snapshot reports that keep change history in `amzn__history`. Equals the REPORT_TYPES keys @@ -883,38 +873,6 @@ AMAZON_TABLES = AMAZON_TABLES + tuple( # Local projections supplement measured vendor columns without changing the table contract. AMAZON_FIELDS = { - _AMZN + "competitor_snapshots": tuple( - {"name": name, "type": kind, "label": label, "source": "competitor_run." + name} - for name, kind, label in ( - ("asin", "VARCHAR", "ASIN"), - ("snapshot_date", "DATE", "Snapshot date"), - ("product_type", "VARCHAR", "Product type"), - ("brand", "VARCHAR", "Brand"), - ("title", "VARCHAR", "Title"), - ("bullets", "VARCHAR", "Bullet points"), - ("image_variants", "VARCHAR", "Image variants"), - ("price", "DOUBLE", "Price"), - ("currency", "VARCHAR", "Currency"), - ("prime", "BOOLEAN", "Prime"), - ("offers", "BIGINT", "Offers"), - ("sales_rank", "BIGINT", "Sales rank"), - ("display_group", "VARCHAR", "Display group"), - ("rank_cat", "VARCHAR", "Rank category"), - ("image", "VARCHAR", "Image"), - ) - ), - _AMZN + "competitor_images": tuple( - {"name": name, "type": kind, "label": label, "source": "competitor_run." + name} - for name, kind, label in ( - ("asin", "VARCHAR", "ASIN"), - ("variant", "VARCHAR", "Variant"), - ("link", "VARCHAR", "Link"), - ("sha256", "VARCHAR", "SHA256"), - ("width", "BIGINT", "Width"), - ("height", "BIGINT", "Height"), - ("snapshot_date", "DATE", "Snapshot date"), - ) - ), _AMZN + "keyword_ranks": tuple( {"name": name, "type": kind, "label": label, "source": source} for name, kind, label, source in ( @@ -3637,9 +3595,6 @@ AMAZON_DEFAULT_COLUMNS = { "cart_adds", "purchases", "backend_keywords", "source_basis"), _AMZN + "keyword_ranks": ("keyword", "date", "organic_rank", "sponsored_rank", "sponsored_rank_basis", "sku", "asin"), - _AMZN + "competitor_snapshots": ("asin", "snapshot_date", "brand", "title", "product_type", - "price", "currency", "prime", "offers", "sales_rank", "rank_cat"), - _AMZN + "competitor_images": ("asin", "variant", "link", "sha256", "width", "height", "snapshot_date"), } # A history table opens on its source's columns plus the validity dates. An undeclared source stays undeclared. AMAZON_DEFAULT_COLUMNS.update({_AMZN + _t + "_history": (*AMAZON_DEFAULT_COLUMNS[_AMZN + _t], "valid_from", "valid_to") @@ -3767,7 +3722,7 @@ AMAZON_GROUPS = { **{_AMZN + t: "Advertising" for t in ( "brand_analytics", "search_query_performance", "search_catalog_performance", "market_basket", "repeat_purchase", "promotion_performance", "coupon_performance", - "sku_keywords", "keyword_ranks", "competitor_snapshots", "competitor_images")}, + "sku_keywords", "keyword_ranks")}, } AMAZON_GROUPS.update({_AMZN + _t + "_history": AMAZON_GROUPS[_AMZN + _t] for _t in AMAZON_HISTORY_SOURCES}) diff --git a/platform/core/external_tools.py b/platform/core/external_tools.py index 991e7aa5c67831c7fb3a9a0f2d3589be43d22ec3..947192850271c25735a301380855545465237c70 100644 --- a/platform/core/external_tools.py +++ b/platform/core/external_tools.py @@ -267,7 +267,7 @@ _APPLICATION = { "templateKind": {"enum": ["route", "catalog", "competitor_watch", "ranking_diagnostic", "amazon_operator", "restock_planner", "reimbursements", "aged_stock", "opportunity_finder", "returns_refunds", "repeat_buyers", "payouts", "account_health", - "amazon_dashboard", "amazon_profit", "alerts", "reports"]}, + "amazon_dashboard", "amazon_profit", "alerts"]}, "fieldMap": {"type": "object", "additionalProperties": _STRING}, "navPlacement": _object({"section": {"const": "app"}, "order": {"type": "integer", "minimum": 0, "maximum": 999}}, diff --git a/platform/core/shares.py b/platform/core/shares.py index 014886f99889d043f1e8541870d2c0b017fac5f1..c2cf3b0d4e20f03213787f4f9014d2cebc571ab8 100644 --- a/platform/core/shares.py +++ b/platform/core/shares.py @@ -80,7 +80,7 @@ SHARES_KEY = 'object_shares' #: here is this tuple member. That is the point of one registry: the new object's WALL is written #: once in `core.perm_scope`, and its DOORS once in `routes_shares.py` — never a fourth #: permission decision in a fourth file ([[one-evaluator-per-question]]). -KINDS = ('view', 'folder', 'database', 'field', 'cohort', 'workbook', 'report') +KINDS = ('view', 'folder', 'database', 'field', 'cohort', 'workbook') #: `*` is "everyone who can already open the surface". It is NOT "every account on the platform". #: Spelled as a single character so it can never collide with a username (usernames are lower-case @@ -692,32 +692,13 @@ def reconcile_database_grants(st=None, *, tenant=None, table_keys=None): return repaired[0] -def _synthetic_qa_publication(target, table_keys): - """Identify private scratch publications outside the legacy account plane.""" - from core import user_tables, users - - qa = users.qa_identity(users._QA_USERNAME) - if (not qa or not table_keys or getattr(target, 'key', None) != qa['tenant'] - or getattr(getattr(target, 'tenant', None), 'key', None) != qa['tenant']): - return False - for key in table_keys: - if not isinstance(key, str) or not key.startswith(user_tables.KEY_PREFIX): - return False - definition = user_tables.get_definition(key, st=target) - if (not isinstance(definition, dict) or definition.get('key') != key - or definition.get('createdBy') != qa['username'] - or definition.get('source') != user_tables.BLANK_SOURCE): - return False - return True - - def _strict_accounts(): """Every account record, read strictly: the account rows on PostgreSQL, else the users document.""" from core import account_rows, users # On PostgreSQL the users document is a layout marker; reading it repairs no account at all. if account_rows._pg_source(users.store): return dict(account_rows._all(users.store)) - registry = users.registry() + registry = users.store._read_strict('users') if account_rows.is_marker(registry): raise RuntimeError('the account registry is unavailable') return registry @@ -727,9 +708,6 @@ def repair_published_database_grants(st=None, *, table_keys): """Attempt one repair after publication, deferring failures to boot/editor except freeze refusal.""" from core import freeze_notice try: - table_keys = tuple(table_keys) - if _synthetic_qa_publication(_st(st), table_keys): - return 0 return reconcile_database_grants(st, table_keys=table_keys) except freeze_notice.DataInputFrozen: raise diff --git a/platform/harness/amazon_client.py b/platform/harness/amazon_client.py index e23212c763efad0f852ff17ec45788958e394af6..659d6d4d4df606b55bc9d90a3236decf8d1ee5bb 100644 --- a/platform/harness/amazon_client.py +++ b/platform/harness/amazon_client.py @@ -450,11 +450,8 @@ REPORT_TYPES = { "report": "GET_BRAND_ANALYTICS_SEARCH_QUERY_PERFORMANCE_REPORT", "history": {"from": "2025-04-01", "evidence": "Monthly FATAL, the period may fall outside the available data range, for 2021-09, " - "2024-10, 2024-11, 2025-01 and 2025-03; 2025-04 onward landed (2026-11-14)", - "periods": {"WEEK": {"from": "2025-07-13", "provisional": True, - "evidence": "2026-11-14 probe refused 2025-07-06 and served 2025-10-05. " - "The intervening twelve weeks are unasked. 2025-07-13 is the " - "first unasked Sunday, not a proved earliest served floor"}}}, + "2024-10, 2024-11, 2025-01 and 2025-03; 2025-04 onward landed. Weekly FATAL through the " + "week of 2025-07-06, DONE 2025-10-05 (2026-11-14)"}, "role": "Brand Analytics", "validates_against": "Amazon dataByAsin row count and ASIN purchases per batch vs stored rows for the latest month", "json_root": "dataByAsin", "monthly": True, diff --git a/platform/harness/amazon_schema.py b/platform/harness/amazon_schema.py index 3bb62ba261fee6cdf8f38e2e32538e815988f0c0..aee3e372d128fbbedf4f630973a3c3cd07d97bfe 100644 --- a/platform/harness/amazon_schema.py +++ b/platform/harness/amazon_schema.py @@ -95,8 +95,6 @@ PROVENANCE = { "search_query_performance": "measured_at_sync", "sku_keywords": "derived", "keyword_ranks": "apify_dataset", - "competitor_snapshots": "competitor_run", - "competitor_images": "competitor_run", "fee_estimates_api": "measured_at_sync", } @@ -107,13 +105,10 @@ PROVENANCE.update(dict.fromkeys((t + "_history" for t in connector_spec.AMAZON_H REVIEW_UNAVAILABLE_NOTE = ( "Amazon's API does not supply written reviews, average star rating or review count. " "Read them on the product page or in Seller Central.") -SQP_LIMIT_NOTE = "Amazon lists at most 100 searches a month for each product." #: Human copy for the disclosure screen, per provenance. ⛔ NO EM DASH OR EN DASH: this reaches a #: screen and standing rule 2 governs it. PROVENANCE_COPY = { - "competitor_run": ("Daily competitor facts saved by the competitor run. Every snapshot day is kept. " - "Image rows keep each distinct variant and link, dated when first saved."), "apify_dataset": ("Apify Amazon search dataset joined to each SKU's latest matching keyword row on or before the capture date. " "Sponsored rank uses the vendor rank or ordered sponsored placements in the captured dataset. " "Global position is separate. Unmatched observations remain in the raw mirror."), @@ -308,8 +303,8 @@ def catalogue(): descriptor = descriptors[key] label, native = descriptor.label, descriptor.native table[key] = {"label": label, "native": native} - if provenance in {"derived", "apify_dataset", "competitor_run"}: - table["columns"] = [dict(field) for field in AMAZON_FIELDS.get(key, ()) + if provenance in {"derived", "apify_dataset"}: + table["columns"] = [dict(field) for field in AMAZON_FIELDS[key] if field["name"] != STORE] table["note"] = PROVENANCE_COPY[provenance] table["local_columns"] = [dict(field) for field in AMAZON_FIELDS.get(key, ()) @@ -319,7 +314,7 @@ def catalogue(): def columns_for(table): """The pinned column list for one table, as `[(name, type)]`. Empty when measured at sync.""" - if PROVENANCE.get(str(table)) in {"derived", "apify_dataset", "competitor_run"}: + if PROVENANCE.get(str(table)) in {"derived", "apify_dataset"}: return [] spec = (catalogue().get("tables") or {}).get(str(table)) or {} return [(c["name"], c.get("type") or "VARCHAR") for c in (spec.get("columns") or [])] @@ -382,17 +377,17 @@ def _table_spec(table, identity, relationships=(), *, grain="entity", line_discr fields = AMAZON_FIELDS.get(DATABASE_PREFIX + table, ()) spec["columns"].extend(dict(field) for field in fields if field["name"] != STORE) spec["local_columns"] = [dict(field) for field in fields if field["name"] == STORE] - local_sources = { - "derived": ("listing_items", "search_query_performance"), - "apify_dataset": ("apify:khadinakbar/amazon-search-scraper", "sku_keywords"), - "competitor_run": ("amazon_rivals.run_quotes", "amazon_quotes_loop.run_tenant"), - } - provenance = PROVENANCE[table] - if provenance in local_sources: + if PROVENANCE[table] == "derived": + spec["columns_source"] = { + "path": "platform/core/connector_spec.py", "reader": "connector_spec.AMAZON_FIELDS", + "provenance": "derived", "assumption": PROVENANCE_COPY["derived"], + "sources": ("listing_items", "search_query_performance"), + } + if PROVENANCE[table] == "apify_dataset": spec["columns_source"] = { "path": "platform/core/connector_spec.py", "reader": "connector_spec.AMAZON_FIELDS", - "provenance": provenance, "assumption": PROVENANCE_COPY[provenance], - "sources": local_sources[provenance], + "provenance": "apify_dataset", "assumption": PROVENANCE_COPY["apify_dataset"], + "sources": ("apify:khadinakbar/amazon-search-scraper", "sku_keywords"), } return spec @@ -575,13 +570,9 @@ TABLE_SPECS.update({ "browse_node_review_topics": _table_spec("browse_node_review_topics", ("browse_node_id", "marketplace_id", "sort_by", "sentiment", "topic"), grain="browse_node_sort_sentiment_topic"), "sku_keywords": _table_spec("sku_keywords", ("keyword_id",), - (("sku", "listing_items", "sku", "many_to_one"),), grain="sku_period_asin_query"), + (("sku", "listing_items", "sku", "many_to_one"),), grain="sku_month_asin_query"), "keyword_ranks": _table_spec("keyword_ranks", ("rank_id",), (("keyword_id", "sku_keywords", "keyword_id", "many_to_one"),), grain="keyword_placement_per_sku"), - "competitor_snapshots": _table_spec("competitor_snapshots", ("asin", "snapshot_date"), - grain="asin_day"), - "competitor_images": _table_spec("competitor_images", ("asin", "variant", "link"), - grain="asin_variant_link"), }) # A history table is one row per version, identified by its version key and first valid date. diff --git a/platform/harness/amazon_store.py b/platform/harness/amazon_store.py index 67b28d1ef525ab432cddcf99bec5ae3830761134..64c9344b38b9b8ee47d40983564a96d79f75e10e 100644 --- a/platform/harness/amazon_store.py +++ b/platform/harness/amazon_store.py @@ -57,7 +57,6 @@ import threading import uuid from contextlib import closing, contextmanager from datetime import datetime, timedelta, timezone -from decimal import Decimal, InvalidOperation from pathlib import Path _HERE = Path(__file__).resolve().parent @@ -92,7 +91,7 @@ DERIVED_TABLES = ("sku_keywords", "keyword_ranks") #: Snapshot reports whose changes are kept as versions in `amzn__history`. HISTORY_SOURCES = tuple(connector_spec.AMAZON_HISTORY_SOURCES) HISTORY_TABLES = tuple(t + "_history" for t in HISTORY_SOURCES) -DATABASE_TABLES = TABLES + DERIVED_TABLES + HISTORY_TABLES + ("competitor_snapshots", "competitor_images") +DATABASE_TABLES = TABLES + DERIVED_TABLES + HISTORY_TABLES PERIOD_TABLES = MARGIN_TABLES + GROUP_A_TABLES LEGACY_SNAPSHOT_TABLES = ("listings", "inventory") @@ -2021,35 +2020,18 @@ def replace_keys_for(table): if c not in _ORIGIN_TYPES) -@contextmanager -def _report_transaction(con): - con.execute("BEGIN TRANSACTION") - try: - yield - con.execute("COMMIT") - except Exception: - con.execute("ROLLBACK") - raise - - -def replace_window(con, table, cols, types, rows, keys, after=None, before=None): +def replace_window(con, table, cols, types, rows, keys, after=None): """Replace incoming key groups atomically, or report why only insertion is possible. `after(con)` runs inside the same transaction, after the insert and before the commit. """ missing = [c for c in keys if c not in cols] - if missing or not keys or not rows: + if missing: note = (f"{table}: window replacement unavailable; missing identity columns: " - f"{', '.join(missing)}; rows inserted; use a report carrying these columns") if missing else None - if before is None and after is None: - return _replace(con, table, cols, types, rows), note - with _report_transaction(con): - if before is not None: - before(con) - count = _replace(con, table, cols, types, rows) - if after is not None: - after(con) - return count, note + f"{', '.join(missing)}; rows inserted; use a report carrying these columns") + return _replace(con, table, cols, types, rows), note + if not keys or not rows: + return _replace(con, table, cols, types, rows), None quote = lambda name: '"' + str(name).replace('"', '""') + '"' target = quote(table) # The window's rows are staged in batches inside the transaction, typed as the mirror and @@ -2060,9 +2042,8 @@ def replace_window(con, table, cols, types, rows, keys, after=None, before=None) try: predicate = " AND ".join( f'{target}.{quote(c)} IS NOT DISTINCT FROM "k".{quote(c)}' for c in keys) - with _report_transaction(con): - if before is not None: - before(con) + con.execute("BEGIN TRANSACTION") + try: count = _replace(con, temporary, cols, types, rows, source_table=table) inserted = cols if table == table_name("brand_analytics") and {"_period_start", "_period_end"} <= set(keys): @@ -2074,6 +2055,10 @@ def replace_window(con, table, cols, types, rows, keys, after=None, before=None) con.execute(f"INSERT INTO {target} ({inserted_columns}) SELECT {inserted_columns} FROM {temporary}") if after is not None: after(con) + con.execute("COMMIT") + except Exception: + con.execute("ROLLBACK") + raise return count, None finally: con.execute(f"DROP TABLE IF EXISTS {temporary}") @@ -2617,11 +2602,7 @@ def sqp_progress_sentence(detail): if any(type(n) is not int or n < 0 for n in steps): raise ValueError("Invalid saved Search Query Performance progress") months_done, months, done, total, eta = steps - weeks_done, weeks = progress.get("weeksDone", 0), progress.get("weeksTotal", 0) - if any(type(n) is not int or n < 0 for n in (weeks_done, weeks)): - raise ValueError("Invalid saved Search Query Performance weekly progress") - weekly = f", {weeks_done:,} of {weeks:,} weeks complete" if weeks else "" - sentence = f"{months_done:,} of {months:,} months complete{weekly}, {done:,} of {total:,} batches." + sentence = f"{months_done:,} of {months:,} months complete, {done:,} of {total:,} batches." if eta: hours, minutes = round(eta / 3600), max(1, round(eta / 60)) left = (f"{hours:,} hours" if hours >= 2 else "1 minute" if minutes == 1 else f"{minutes} minutes") @@ -2649,7 +2630,7 @@ def _sqp_month(con, start, end): "period": period, "start": _utc(start).date().isoformat(), "end": (_utc(end) - timedelta(days=1)).date().isoformat(), - "batches": {}, "asins": {}, "refusedAsins": {}, "retries": {}} + "batches": {}, "asins": {}, "retries": {}} def _sqp_save_month(con, start, marker): @@ -2658,97 +2639,17 @@ def _sqp_save_month(con, start, marker): def _sqp_done(con, start, end, batch): - return not _sqp_missing_asins(_sqp_month(con, start, end), batch.split()) - - -def _sqp_missing_asins(marker, asins): - """Successful ASIN summaries and durable named exclusions complete a period independently of batching.""" - done = set(marker["asins"]) | set(marker.get("refusedAsins", {})) - for batch, retry in marker["retries"].items(): - done.update(set(batch.split()) - set(retry["asins"])) - return [asin for asin in asins if asin not in done] - - -def _sqp_owed_tasks(con, tasks): - """Batch only each period's missing ASINs, retaining unfinished retry identities across catalogue changes.""" - windows = {} - for task in tasks: - if task[0] == SQP_TABLE: - windows.setdefault(task[1:3], []).extend(task[3].split()) - result = [] - for task in tasks: - if task[0] != SQP_TABLE: - result.append(task) - continue - asins = windows.pop(task[1:3], None) - if asins is None: - continue - marker = _sqp_month(con, *task[1:3]) - missing = _sqp_missing_asins(marker, list(dict.fromkeys(asins))) - remaining, batches = set(missing), [] - for batch, retry in marker["retries"].items(): - requested = set(retry["asins"]) - if requested and requested <= remaining: - batches.append(batch) - remaining.difference_update(requested) - batches.extend(sqp_batches([asin for asin in missing if asin in remaining], - _client().REPORT_TYPES[SQP_TABLE]["asin_option_max_chars"])) - result.extend((SQP_TABLE, *task[1:3], batch) for batch in batches) - return result + return batch in _sqp_month(con, start, end)["batches"] def _sqp_week_tasks(start, end, batches): - """Schedule every completed week newest first through the same ASIN batch queue.""" + """Schedule the latest completed week through the same ASIN batch queue.""" if "WEEK" not in _client().REPORT_TYPES[SQP_TABLE].get("periods", ()): return [] - return [(SQP_TABLE, w0, w1, batch) for w0, w1 in reversed(_week_windows(start, end)) + return [(SQP_TABLE, w0, w1, batch) for w0, w1 in _week_windows(start, end)[-1:] for batch in batches] -def _sqp_history_done(con, end): - """Every admitted period and current catalogue ASIN needs its own durable completion evidence.""" - asins = _sqp_catalogue(con) - tasks = first_load_plan([SQP_TABLE], history_floor(con, SQP_TABLE, end), end, - sqp_asins=asins, starts=history_starts(con, [SQP_TABLE], end)) - return bool(asins) and all(_sqp_done(con, task[1], task[2], task[3]) for task in tasks) - - -def _sqp_progress(con, months, weeks, asins, batches, end): - """Count durable per-ASIN completion separately for admitted MONTH and WEEK windows.""" - from harness.amazon_client import REPORT_SUSTAINED_SECONDS - - progress = {} - for period, windows, unit, label in (("MONTH", months, "month", "%Y-%m"), - ("WEEK", weeks, "week", "%Y-%m-%d")): - floor = history_floor(con, SQP_TABLE, end, period=period) - rows = [] - for w0, w1 in windows: - if w0 < floor: - continue - missing = set(_sqp_missing_asins(_sqp_month(con, w0, w1), asins)) - rows.append({unit: w0.strftime(label), "total": len(batches), - "done": sum(not missing.intersection(batch.split()) for batch in batches)}) - progress[unit + "s"] = rows - progress[unit + "sDone"] = sum(r["done"] >= r["total"] for r in rows) - progress[unit + "sTotal"] = len(rows) - rows = progress["months"] + progress["weeks"] - done, total = sum(r["done"] for r in rows), sum(r["total"] for r in rows) - progress.update(batchesDone=done, batchesTotal=total, - etaSeconds=int((total - done) * REPORT_SUSTAINED_SECONDS)) - return progress if rows and batches else None - - -def _sqp_first_load_status(con, cohort): - """Project current period floors and per-ASIN evidence through the shared SQP progress shape.""" - asins = _sqp_catalogue(con) - spec = _client().REPORT_TYPES[SQP_TABLE] - months = _month_windows(history_floor(con, SQP_TABLE, cohort), max(cohort, datetime.now(timezone.utc))) - weeks = (_week_windows(history_floor(con, SQP_TABLE, cohort, period="WEEK"), cohort) - if "WEEK" in spec.get("periods", ()) else []) - progress = _sqp_progress(con, months, weeks, asins, sqp_batches(asins, spec["asin_option_max_chars"]), cohort) - return {"progress": progress, "sentence": sqp_progress_sentence({"progress": progress})} if progress else None - - class SQPBudgetDeferred(RuntimeError): """A batch or its single retry remains durable for the next bounded pass.""" @@ -2757,42 +2658,6 @@ class SQPPassDeferred(SQPBudgetDeferred): """A scheduled pass spent `SQP_PASS_REPORTS`; only SQP stops, other tables keep running.""" -def _sqp_document_rows(text, spec, requested, marker, meta): - """Validate the source document before flattening rows and computing per-ASIN totals.""" - text = _document_text(text) - doc = json.loads(text) - if not isinstance(doc, dict): - raise ValueError("Search Query Performance document must be a JSON object") - raw = doc.get(spec["json_root"]) - if not isinstance(raw, list) or any(not isinstance(r, dict) for r in raw): - raise ValueError("Search Query Performance document has no dataByAsin row array") - totals = {asin: {"count": 0, "purchases": 0} for asin in requested} - for row in raw: - asin = row.get("asin") - if not isinstance(asin, str): - raise ValueError("Search Query Performance ASIN must be a string") - if asin not in totals or row.get("startDate") != marker["start"] or row.get("endDate") != marker["end"]: - raise ValueError("Search Query Performance returned an ASIN or period outside the requested batch") - purchase_data = row.get("purchaseData") - if not isinstance(purchase_data, dict): - raise ValueError("Search Query Performance purchaseData must be a JSON object") - purchase = purchase_data.get("asinPurchaseCount") - if type(purchase) is not int or purchase < 0: - raise ValueError("Search Query Performance omitted a nonnegative ASIN purchase count") - totals[asin]["count"] += 1 - totals[asin]["purchases"] += purchase - columns, rows = parse_json_report(text, spec["json_root"], - document_id=meta.get("reportDocumentId"), report_type=spec["report"]) - seen = set() - for row in rows: - key = amazon_schema.row_identity_for(SQP_TABLE, row) - if key in seen: - raise ValueError("Search Query Performance returned a duplicate month, ASIN and search query") - seen.add(key) - row["_report_period"] = marker.get("period", "MONTH") - return columns, rows, totals, len(raw) - - def _sync_sqp_batch(con, fields, start, end, batch, limit, request, problem): """One completed period and batch using the shared report transport and pass budget. @@ -2823,7 +2688,6 @@ def _sync_sqp_batch(con, fields, start, end, batch, limit, request, problem): reason = reason.replace("\u2014", "; ").replace("\u2013", " to ") for asin in refused: limit["refusedAsins"][asin] = reason - marker.setdefault("refusedAsins", {})[asin] = reason problem(sqp_refusal_message(asin, reason)) if not refused: limit["errors"] = [reason] @@ -2843,15 +2707,39 @@ def _sync_sqp_batch(con, fields, start, end, batch, limit, request, problem): _sqp_save_month(con, start, marker) return 0 - columns, rows, totals, source_count = _sqp_document_rows(text, spec, requested, marker, meta) + # Read source evidence BEFORE flattening or typing. Missing/malformed JSON is never empty success. + text = _document_text(text) + doc = json.loads(text) + raw = doc.get(spec["json_root"]) + if not isinstance(raw, list) or any(not isinstance(r, dict) for r in raw): + raise ValueError("Search Query Performance document has no dataByAsin row array") + totals = {asin: {"count": 0, "purchases": 0} for asin in requested} + for row in raw: + asin = row.get("asin") + if asin not in totals or row.get("startDate") != marker["start"] or row.get("endDate") != marker["end"]: + raise ValueError("Search Query Performance returned an ASIN or period outside the requested batch") + purchase = (row.get("purchaseData") or {}).get("asinPurchaseCount") + if type(purchase) is not int or purchase < 0: + raise ValueError("Search Query Performance omitted a nonnegative ASIN purchase count") + totals[asin]["count"] += 1 + totals[asin]["purchases"] += purchase + columns, rows = parse_json_report(text, spec["json_root"], + document_id=meta.get("reportDocumentId"), report_type=spec["report"]) identity = amazon_schema.TABLE_SPECS[SQP_TABLE]["row_identity"] + seen = set() + for row in rows: + key = amazon_schema.row_identity_for(SQP_TABLE, row) + if key in seen: + raise ValueError("Search Query Performance returned a duplicate month, ASIN and search query") + seen.add(key) + row["_report_period"] = marker.get("period", "MONTH") columns = list(dict.fromkeys([*columns, *identity, "_report_period", "purchase_data_asin_purchase_count", *_ORIGIN_TYPES])) mirror = table_name(SQP_TABLE) types = _ensure(con, mirror, columns, rows, preserve_rows=True, declared={"purchase_data_asin_purchase_count": "BIGINT"}) marker["asins"].update(totals) - marker["batches"][batch] = {"asins": requested, "count": source_count, + marker["batches"][batch] = {"asins": requested, "count": len(raw), "purchases": sum(t["purchases"] for t in totals.values()), "reportDocumentId": meta["reportDocumentId"]} con.execute("BEGIN TRANSACTION") @@ -2895,24 +2783,6 @@ def _validate_sqp(con): return {"state": "refused", "reason": f"Search Query Performance reconciliation failed: {exc}"} -def _sku_keyword_source_fields(columns): - from harness.amazon_sqp_reads import report_period_sql - - fields = {"start_date": "start_date", "end_date": "end_date", "asin": "asin", - "search_query": "search_query_data_search_query", - "impressions": "impression_data_asin_impression_count", - "clicks": "click_data_asin_click_count", "cart_adds": "cart_add_data_asin_cart_add_count", - "purchases": "purchase_data_asin_purchase_count"} - expressions = [] - for name, source in fields.items(): - numeric = name in ("impressions", "clicks", "cart_adds", "purchases") - kind, cast = ("BIGINT", "TRY_CAST") if numeric else ("VARCHAR", "CAST") - expressions.append(f'{cast}("{source}" AS {kind}) AS {name}' if source in columns else - f'NULL::{kind} AS {name}') - period = report_period_sql(columns) if {"start_date", "end_date"} <= columns else "NULL::VARCHAR" - return ", ".join([*expressions, f"{period} AS _report_period"]) - - def refresh_sku_keywords(con): """Bind a local derived view to listing summaries and raw SQP without another source copy.""" listing_columns = _columns_of(con, "amzn_listing_items") @@ -2940,7 +2810,19 @@ def refresh_sku_keywords(con): output_meta = ', "_connection_id", store' if scoped else "" backend_meta = ', l."_connection_id", l.store' if scoped else "" backend_match = ' AND m."_connection_id" IS NOT DISTINCT FROM l."_connection_id"' if scoped else "" - select = _sku_keyword_source_fields(raw_columns) + raw_fields = { + "start_date": "start_date", "end_date": "end_date", "asin": "asin", + "search_query": "search_query_data_search_query", + "impressions": "impression_data_asin_impression_count", + "clicks": "click_data_asin_click_count", "cart_adds": "cart_add_data_asin_cart_add_count", + "purchases": "purchase_data_asin_purchase_count", + } + select = ", ".join( + f'{"TRY_CAST" if name in ("impressions", "clicks", "cart_adds", "purchases") else "CAST"}' + f'("{source}" AS {"BIGINT" if name in ("impressions", "clicks", "cart_adds", "purchases") else "VARCHAR"}) AS {name}' + if source in raw_columns else + f'NULL::{"BIGINT" if name in ("impressions", "clicks", "cart_adds", "purchases") else "VARCHAR"} AS {name}' + for name, source in raw_fields.items()) raw_from = " FROM amzn_search_query_performance" if raw_columns else " WHERE FALSE" con.execute(f""" CREATE OR REPLACE VIEW amzn_sku_keywords AS @@ -2956,25 +2838,23 @@ def refresh_sku_keywords(con): SELECT asin{account_group}, COUNT(DISTINCT sku) AS mapped_sku_count FROM mapping GROUP BY asin{account_group} ), raw AS (SELECT {select}{raw_meta}{raw_from}), queries AS ( SELECT *, CAST(to_json([start_date, end_date, asin, search_query{grain_meta}]) AS VARCHAR) AS source_grain - FROM raw WHERE _report_period IS NOT NULL + FROM raw ), matched AS ( SELECT l.sku, {query_columns}, l.backend_keywords, c.mapped_sku_count{matched_meta} FROM listings l JOIN mapping m ON m.sku=l.sku{mapping_join} JOIN queries q ON q.asin=m.asin{query_join} JOIN mapped_counts c ON c.asin=m.asin{count_join} ) SELECT md5(CAST(to_json([sku, source_grain]) AS VARCHAR)) AS keyword_id, - sku, asin, CASE WHEN _report_period='MONTH' THEN substr(start_date, 1, 7) || '-01' END AS month, - start_date, end_date, search_query, + sku, asin, substr(start_date, 1, 7) || '-01' AS month, start_date, end_date, search_query, impressions, clicks, cart_adds, purchases, backend_keywords, source_grain, mapped_sku_count, 'Amazon ASIN/query metrics shared across ' || CAST(mapped_sku_count AS VARCHAR) || - ' listing SKUs. No SKU allocation. Aggregate distinct SQP source grains.' AS source_basis, - _report_period{output_meta} + ' listing SKUs. No SKU allocation. Aggregate distinct SQP source grains.' AS source_basis{output_meta} FROM matched UNION ALL SELECT md5(CAST(to_json([l.sku, 'backend_keywords'{', l."_connection_id"' if scoped else ''}]) AS VARCHAR)), l.sku, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, l.backend_keywords, NULL, NULL, - 'Listing backend keywords only. No matching Search Query Performance rows.', NULL::VARCHAR{backend_meta} + 'Listing backend keywords only. No matching Search Query Performance rows.'{backend_meta} FROM listings l WHERE NOT EXISTS (SELECT 1 FROM matched m WHERE m.sku=l.sku{backend_match}) """) refresh_keyword_ranks(con) @@ -3071,56 +2951,43 @@ def _validate_keyword_ranks(con): def _validate_sku_keywords(con): - """Tie distinct derived grains and purchases to raw rows separately for every MONTH and WEEK.""" - from harness.amazon_sqp_reads import _independent_period - + """Compare each month's distinct derived SQP grains with independently grouped raw source rows.""" basis = amazon_schema.PROVENANCE_COPY["derived"] - columns = _columns_of(con, table_name(SQP_TABLE)) - if not columns: + if not _columns_of(con, table_name(SQP_TABLE)): return {"state": "unproven", "reason": "Search Query Performance source is absent", "source_basis": basis} try: - declared = "_report_period" if "_report_period" in columns else "NULL::VARCHAR" - account = ', "_connection_id"' if amazon_schema.CONNECTION_ID in columns else "" - derived_account = ', "_connection_id"' if amazon_schema.CONNECTION_ID in _columns_of(con, "amzn_sku_keywords") else "" - raw = con.execute(f""" - SELECT CAST(start_date AS VARCHAR), CAST(end_date AS VARCHAR), list(DISTINCT {declared}), - COUNT(*), COUNT(DISTINCT (asin, search_query_data_search_query{account})), + raw = con.execute(""" + SELECT substr(CAST(start_date AS VARCHAR), 1, 7), + COUNT(*), COUNT(DISTINCT (start_date, end_date, asin, search_query_data_search_query)), SUM(purchase_data_asin_purchase_count), COUNT(purchase_data_asin_purchase_count), COUNT(*) FILTER (WHERE purchase_data_asin_purchase_count < 0 OR start_date IS NULL OR end_date IS NULL OR asin IS NULL OR search_query_data_search_query IS NULL) - FROM amzn_search_query_performance GROUP BY ALL + FROM amzn_search_query_performance GROUP BY 1 """).fetchall() - derived = con.execute(f""" - SELECT start_date, end_date, _report_period, COUNT(*), SUM(purchases), SUM(invalid) + derived = con.execute(""" + SELECT substr(month, 1, 7), COUNT(*), SUM(purchases), SUM(invalid) FROM ( - SELECT start_date, end_date, _report_period, asin, search_query{derived_account}, MIN(purchases) AS purchases, + SELECT month, start_date, end_date, asin, search_query, MIN(purchases) AS purchases, CASE WHEN COUNT(purchases) <> COUNT(*) OR MIN(purchases) <> MAX(purchases) - OR COUNT(*) <> COUNT(DISTINCT sku) - OR COUNT(*) FILTER (WHERE month IS DISTINCT FROM CASE WHEN _report_period='MONTH' - THEN substr(start_date, 1, 7) || '-01' END) > 0 THEN 1 ELSE 0 END AS invalid + OR COUNT(*) <> COUNT(DISTINCT sku) THEN 1 ELSE 0 END AS invalid FROM amzn_sku_keywords WHERE source_grain IS NOT NULL - GROUP BY ALL - ) GROUP BY ALL + GROUP BY month, start_date, end_date, asin, search_query + ) GROUP BY 1 """).fetchall() - actual = {(start, end, period): {"grains": n, "purchases": purchases, "invalid": invalid} - for start, end, period, n, purchases, invalid in derived} - held, months = {}, {} - for start, end, declarations, count, distinct, purchases, present, invalid in raw: - labels = {_independent_period(start, end, declared) for declared in declarations} - period = labels.pop() if len(labels) == 1 else None - key = (start, end, period) - observed = actual.get(key, {"grains": 0, "purchases": 0, "invalid": 0}) - good = (period is not None and count == distinct == present and not invalid and not observed["invalid"] + actual = {month: {"grains": n, "purchases": purchases, "invalid": invalid} + for month, n, purchases, invalid in derived} + months = {} + for month, count, distinct, purchases, present, invalid in raw: + observed = actual.get(month, {"grains": 0, "purchases": 0, "invalid": 0}) + good = (count == distinct == present and not invalid and not observed["invalid"] and observed["grains"] == distinct and observed["purchases"] == purchases) - held[key] = {"state": "ok" if good else "refused", "start": start, "end": end, "period": period, - "expected": {"grains": distinct, "purchases": purchases}, "actual": observed} - if period == "MONTH": - months[start[:7]] = held[key] - extra = set(actual) - set(held) - good = bool(held) and not extra and all(row["state"] == "ok" for row in held.values()) - return {"state": "ok" if good else "refused" if held or extra else "unproven", - "months": months, "periods": list(held.values()), "source_basis": basis, - "reason": ("Distinct ASIN/query grains reconcile to raw SQP for every period" if good else + months[month] = {"state": "ok" if good else "refused", + "expected": {"grains": distinct, "purchases": purchases}, "actual": observed} + extra = set(actual) - set(months) + good = bool(months) and not extra and all(row["state"] == "ok" for row in months.values()) + return {"state": "ok" if good else "refused" if months or extra else "unproven", + "months": months, "source_basis": basis, + "reason": ("Distinct ASIN/query grains reconcile to raw SQP for every month" if good else "SQP coverage or purchases differ. Unmapped ASINs, duplicate grains and missing counts cannot reconcile")} except Exception as exc: return {"state": "refused", "source_basis": basis, @@ -3156,12 +3023,8 @@ def first_load_plan(tables, start, end, *, recent_days=None, size=MAX_WINDOW_DAY for table in tables: spec = _client().REPORT_TYPES[table] table_start = starts.get(table, start) if starts is not None else start - if table == SQP_TABLE: - rule = _history_rule(spec, "WEEK") - week_start = _date_utc(rule["from"]) if rule.get("from") else table_start - week_start = starts.get((table, "WEEK"), week_start) if starts is not None else week_start - weekly_history.extend(_sqp_week_tasks(week_start, end, sqp_batches( - sqp_asins, spec["asin_option_max_chars"]))) + if table == SQP_TABLE and table_start < end: + recent.extend(_sqp_week_tasks(table_start, end, sqp_batches(sqp_asins, spec["asin_option_max_chars"]))) if table == "settlements": continue if spec.get("snapshot"): @@ -4096,13 +3959,6 @@ def standing_refusals(report): return sorted(holds["standing"]) if holds else [] -def _deferred_table(reason, refusal, *, current=False): - """Keep a seller-wide refusal visible when a scheduling hold prevents another request.""" - if refusal and refusal.get("state") == "not_offered": - return {"state": "refused", "amazonStatus": "NOT_OFFERED", "reason": refusal["reason"]} - return {"state": "deferred", "reason": reason, "current": current} - - class _AmazonMirrorStore(_mirrors.MirrorStore): def __init__(self): self.connector, self.spec = "amazon", _mirror_spec("amazon") @@ -4180,10 +4036,10 @@ class _AmazonMirrorStore(_mirrors.MirrorStore): """ rows = report.get("tables") or {} problems = [str(problem) for key in ("problems", "audit_problems") for problem in report.get(key) or []] - # Landed SQP batches and dated periods publish even while older history stays owed. + # Landed SQP batches and landed months count as finished work, so a chunk of only that progress still publishes. if report.get("state") in {"partial", "failed"} and not any( isinstance(row, dict) and (row.get("state") != "deferred" or name == SQP_TABLE - or bool(row.get("landedMonths") or row.get("landedWeeks"))) + or bool(row.get("landedMonths"))) and _mirrors._table_sync_complete("amazon", row) for name, row in rows.items()): return None unfinished = {name[len(PREFIX):] if name.startswith(PREFIX) else name for name in waiting} @@ -4208,7 +4064,7 @@ class _AmazonMirrorStore(_mirrors.MirrorStore): unfinished |= set(partner) for names in scoped.values(): names |= {pairs[name] for name in list(names) if name in pairs and pairs[name] not in unfinished} - if not unfinished and not scoped and not problems and report.get("state") not in {"partial", "failed"}: + if not unfinished and not scoped and not problems: return None owners = {key for name in set(rows) | unfinished for key in (name, table_name(name))} account_owned = {str(problem) for cid, result in accounts.items() @@ -5255,10 +5111,8 @@ def _restore_first_load_wait(tenant_key, con): return due -def _loaded_refresh_hold(con, table, now, *, slot_pass=False, history=False): +def _loaded_refresh_hold(con, table, now, *, slot_pass=False): """Return a loaded table's cadence hold while its first load continues, or None when due.""" - if history and table == "brand_analytics" and os.environ.get("AIOS_AMAZON_WEEKLY_HISTORY") == "1": - return None now = _utc(now) tick = _read_state(con, entity_key("sync_tick_" + table)) # A queued resync is a repair: only Amazon's own floor holds it. @@ -5550,8 +5404,6 @@ def needs_first_load(tenant_key, rt=None, *, completed_tables=None, repair_table floor = read_backfill_floor(con, table) if cohort is None: return False - if table == SQP_TABLE: - return _sqp_history_done(con, cohort) # The same per-table floor first_load_plan uses, so a load that lands every window completes. return floor is not None and floor <= history_floor(con, table, cohort) # ⛔ HOTFIX 2026-09-29 (D-3496): the same rule for a load that STOPPED. A load left @@ -5621,8 +5473,6 @@ def first_load_status(tenant_key, rt=None): "nextDue": _iso(_first_load_due(con)) if _first_load_due(con) else None} if job: out["job"] = job - if SQP_TABLE in enabled_tables: - out["sqp"] = _sqp_first_load_status(con, cohort) return out finally: con.close() @@ -5697,6 +5547,34 @@ def _doc(docs, name): return value if isinstance(value, dict) else {} +#: D-17410: live's daily Amazon pass runs from 00:00Z to about 02:00Z, and staging's chain shares its seller quota. +LIVE_DAILY_HOLD_START_HOUR = 0 +LIVE_DAILY_HOLD_END_HOUR = 3 +#: UTC hour buckets each seller's ledger row keeps, so both stores' ledgers show spend per hour. +REPORT_LEDGER_HOURS_KEPT = 72 +_LIVE_HOLD_NOTICED = set() + + +def live_daily_hold_seconds(now=None): + """Seconds staging's report requests wait for live's daily Amazon pass, else 0.""" + if os.environ.get("AIOS_FIRST_LOAD_TARGET") != "staging": + return 0.0 + at = datetime.fromtimestamp(time.time() if now is None else float(now), timezone.utc) + day = at.replace(hour=0, minute=0, second=0, microsecond=0) + start = day + timedelta(hours=LIVE_DAILY_HOLD_START_HOUR) + stop = day + timedelta(hours=LIVE_DAILY_HOLD_END_HOUR) + return (stop - at).total_seconds() if start <= at < stop else 0.0 + + +def _note_live_hold(stamp): + window = datetime.fromtimestamp(float(stamp), timezone.utc).strftime("%Y-%m-%d") + if window in _LIVE_HOLD_NOTICED: + return + _LIVE_HOLD_NOTICED.add(window) + print(f"[amazon-report-ledger] staging holds its report requests until {LIVE_DAILY_HOLD_END_HOUR:02d}:00Z " + "while live's daily Amazon pass runs (D-17410)", flush=True) + + def take_report_token(docs, seller, *, now=None, burst=None, refill_seconds=None): """Draw one createReport token for `seller` from the store ledger. -> `(granted, wait seconds)`. @@ -5704,6 +5582,11 @@ def take_report_token(docs, seller, *, now=None, burst=None, refill_seconds=None """ from harness.amazon_client import REPORT_BURST, REPORT_SUSTAINED_SECONDS stamp = time.time() if now is None else float(now) + hold = live_daily_hold_seconds(stamp) + if hold > 0: + # A held draw spends no token and leaves the ledger document unchanged. + _note_live_hold(stamp) + return False, hold burst = float(REPORT_BURST if burst is None else burst) refill = float(REPORT_SUSTAINED_SECONDS if refill_seconds is None else refill_seconds) out = {} @@ -5716,6 +5599,10 @@ def take_report_token(docs, seller, *, now=None, burst=None, refill_seconds=None if tokens >= 1.0: tokens -= 1.0 row.update(spent=int(row.get("spent") or 0) + 1, lastCreate=stamp) + hours = dict(row.get("hours") or {}) + hour = datetime.fromtimestamp(stamp, timezone.utc).strftime("%Y-%m-%dT%H") + hours[hour] = int(hours.get(hour) or 0) + 1 + row["hours"] = {key: hours[key] for key in sorted(hours)[-REPORT_LEDGER_HOURS_KEPT:]} out.update(granted=True, wait=0.0) else: out.update(granted=False, wait=(1.0 - tokens) * refill) @@ -5739,6 +5626,20 @@ def first_load_job_live(docs, *, now=None): return False +#: run_first_load_generation's default holder prefix. A resident worker's own pass holds the lease as `:amazon:`. +FIRST_LOAD_HOLDER_PREFIX = "amazon-first-load-" + + +def first_load_job_holds(docs, *, now=None): + """True while a first-load generation, never a resident worker, holds a live Amazon writer lease.""" + try: + doc = _doc(docs, AMAZON_WRITER_LEASE_DOC) + return (_lease_live(doc, time.time() if now is None else now) + and str(doc.get("holder") or "").startswith(FIRST_LOAD_HOLDER_PREFIX)) + except Exception: # noqa: BLE001 + return False + + def acquire_amazon_lease(docs, holder, *, now=None, ttl=FIRST_LOAD_LEASE_SECONDS): """Take or renew the Amazon-scoped writer lease. True when `holder` holds it. The Odoo lease is untouched.""" stamp = time.time() if now is None else float(now) @@ -5773,6 +5674,11 @@ def _ledger_draw(docs): granted, wait = take_report_token(docs, cx.seller_key(fields)) except Exception: # noqa: BLE001 return # An unreadable ledger leaves the in-process pacer in charge. + if not granted and live_daily_hold_seconds() > 0: + raise cx.AmazonThrottled( + f"Staging holds its Amazon report requests until {LIVE_DAILY_HOLD_END_HOUR:02d}:00 UTC while " + "live's daily Amazon pass runs. This report resumes after the hold.", + retry_after=wait, operation="createReport") if not granted and first_load_job_live(docs): raise cx.AmazonThrottled( "The first-load job is using this seller's Amazon report quota. " @@ -6444,22 +6350,9 @@ def first_load_job_summary(docs, *, table_status=None): "remaining": record.get("remaining", last.get("remainingAfter")), "tablesPublished": list(last.get("tablesPublished") or []), "end": last.get("end"), "operation": last.get("operation"), "resume_at": last.get("resume_at"), - "sqp": last.get("sqp"), "tables": {**(last.get("tables") or {}), **(table_status or {})}} -def _checkpoint_first_load_sqp(tenant_key, rt, cohort, record, save): - """Checkpoint derived SQP progress after publication without making it another completion authority.""" - if cohort is None or not read_database_enablement(rt)["databases"].get(amazon_schema.DATABASE_PREFIX + SQP_TABLE): - return - con, _path = _connect_tenant_mirror(tenant_key, rt) - try: - record["sqp"] = _sqp_first_load_status(con, cohort) - finally: - con.close() - save(lambda doc: doc.update(sqp=record["sqp"])) - - def _record_generation(docs, row): def add(doc): rows = [r for r in (doc.get("generations") or []) if r.get("generation") != row["generation"]] @@ -6469,18 +6362,6 @@ def _record_generation(docs, row): docs.update(FIRST_LOAD_GENERATIONS_KEY, add, flush="sync") -def _record_brand_week_write(record, task, wall_seconds, cpu_seconds, rows): - """Accumulate measured writes while retaining each week's own CPU and wall time.""" - timing = record.setdefault("brandAnalytics", {"wallSeconds": 0, "cpuSeconds": 0, "periods": []}) - week = {"account": task["account"], "_period_start": task["start"][:10], - "_period_end": task["end"][:10], "wallSeconds": round(wall_seconds, 3), - "cpuSeconds": round(cpu_seconds, 3)} - timing["periods"].append(week) - for field in ("wallSeconds", "cpuSeconds"): - timing[field] = round(sum(item[field] for item in timing["periods"]), 3) - timing["rows"] = rows - - def run_first_load_generation(tenant_key, rt=None, *, generation=1, budget_seconds=5 * 3600, log=print, docs=None, plan=None, client=None, write=None, delegate=None, counts=None, holder=None, now=time.time, sleep=time.sleep, poll_seconds=None, job_id=None): @@ -6546,13 +6427,12 @@ def run_first_load_generation(tenant_key, rt=None, *, generation=1, budget_secon if cursor.get("end") and end and cursor["end"] != _iso(end): # A new cohort end means a new first load; the old cursor's windows describe another plan. cursor = {} - cohort = end or (_utc(cursor["end"]) if cursor.get("end") else None) finished = dict(cursor.get("finished") or {}) opened = dict(cursor.get("open") or {}) throttled = dict(cursor.get("throttled") or {}) failed = dict(cursor.get("failed") or {}) downloads = dict(cursor.get("downloads") or {}) - save(lambda doc: doc.update(tenant=key, end=_iso(cohort) if cohort else None, backfillDays=DEFAULT_BACKFILL_DAYS, + save(lambda doc: doc.update(tenant=key, end=_iso(end) if end else None, backfillDays=DEFAULT_BACKFILL_DAYS, generation=int(generation), finished=finished, open=opened, throttled=throttled, failed=failed, downloads=downloads)) @@ -6561,7 +6441,7 @@ def run_first_load_generation(tenant_key, rt=None, *, generation=1, budget_secon record["remainingBefore"] = sum(owed(t) for t in tasks) mine = _pace_weekly_history([t for t in tasks if t.get("prefetch")], owed) - # A write that finishes no window waits for the next generation, including unserved documents. + # A window the writer served but did not finish (deferred, held) waits for the next generation. stalled = set() own_tables = sorted({t["table"] for t in mine if owed(t)}) ready = {} @@ -6678,8 +6558,6 @@ def run_first_load_generation(tenant_key, rt=None, *, generation=1, budget_secon prefix.append(task) if not prefix: continue - if table == "brand_analytics": - prefix = prefix[:1] # Each week needs its own measured write interval. sibling = SHARED_REPORT_SIBLINGS.get(table) pair = [t for t in by_table.get(sibling, []) if owed(t)][:len(prefix)] if sibling else [] units = [table] + ([sibling] if pair and [t["key"] for t in pair] == [t["key"] for t in prefix] @@ -6692,30 +6570,28 @@ def run_first_load_generation(tenant_key, rt=None, *, generation=1, budget_secon log(f"{table}: the first-load write failed ({type(exc).__name__}); " "its windows stay owed and the next generation downloads them again.") for t in prefix: - stalled.add(t["id"]) ready.pop(t["key"], None) continue - wall_seconds, cpu_seconds = time.time() - t0, time.process_time() - c0 + if table == "brand_analytics": + record["brandAnalytics"] = {"wallSeconds": round(time.time() - t0, 1), + "cpuSeconds": round(time.process_time() - c0, 1), + "rows": (result["tables"].get(table) or {}).get("rows")} stamp_done = _epoch_iso(now()) for unit in units: row = result["tables"].get(unit) or {} - ok = result["published"] and (row.get("state", "done") not in { + ok = result["published"] and row.get("state", "done") not in { "failed", "refused", "throttled", "pending", "held", "deferred", "disabled"} - or unit == "brand_analytics" and row.get("state") == "deferred" - and row.get("current") and row.get("landedWeeks", 0) > 0) if not ok: continue for t in (prefix if unit == table else pair): if t["key"] in result["served"]: - if unit == "brand_analytics": - _record_brand_week_write(record, t, wall_seconds, cpu_seconds, row.get("rows")) finished[t["id"]] = {"at": stamp_done, "generation": int(generation), "reportId": (opened.get(t["key"]) or {}).get("reportId")} record["windowsWritten"] += 1 if unit not in record["tablesPublished"]: record["tablesPublished"].append(unit) for t in prefix + pair: - if t["id"] not in finished: + if t["id"] not in finished and t["key"] in result["served"]: stalled.add(t["id"]) ready.pop(t["key"], None) if t["id"] in finished or t["key"] not in result["served"]: @@ -6755,8 +6631,6 @@ def run_first_load_generation(tenant_key, rt=None, *, generation=1, budget_secon break record["otherTablesPasses"] += 1 result = result if isinstance(result, dict) else {} - if plan is None: - _checkpoint_first_load_sqp(key, rt, cohort, record, save) if result.get("state") != "partial" or not result.get("landedTasks"): break try: @@ -6765,16 +6639,12 @@ def run_first_load_generation(tenant_key, rt=None, *, generation=1, budget_secon after = tasks record["remainingAfter"] = sum(t["id"] not in finished and not t.get("landed") and t["id"] not in failed for t in after) - record["brandAnalyticsPeriods"] = sorted({t["start"][:10] for t in tasks + after - if t["table"] == "brand_analytics" and t.get("start") - and (t.get("landed") or t["id"] in finished)}) try: row_counts = counts(list(dict.fromkeys(EMPTY_REASON_TABLES + BEHIND_TABLES))) except Exception: # noqa: BLE001 row_counts = {} table_status = None if plan is None: - _checkpoint_first_load_sqp(key, rt, cohort, record, save) con, _path = _connect_tenant_mirror(key, rt) try: table_status = {"inventory": inventory_table_status(con)} @@ -6816,129 +6686,6 @@ _AMAZON_FLOOR_TEXT = re.compile( _FLOOR_MIN_AGE = timedelta(days=45) -def _sqp_depth_start(value, period): - """Admit a complete MONTH or Sunday-based WEEK boundary.""" - if period not in _client().SQP_PERIOD: - raise ValueError("SQP depth supports MONTH and WEEK only") - if isinstance(value, str) and re.fullmatch(r"\d{4}-\d{2}-\d{2}", value): - value += "T00:00:00Z" - day = _utc(value) - boundary = _grain_floor(day, period.lower()) - if day != boundary: - raise ValueError("SQP depth needs the first day of a complete period") - return day - - -def _sqp_depth_shift(start, period, count): - if period == "WEEK": - return start + timedelta(weeks=count) - return _months_before(start, -count) - - -def _sqp_depth_eligible(row, now): - """Only Amazon's old out-of-range refusal can bound served history.""" - return (row["state"] == "out_of_range" and row.get("amazonStatus") in ("FATAL", 400) - and bool(_AMAZON_FLOOR_TEXT.search(row.get("amazon") or "")) - and _date_utc(row["end"]) + timedelta(days=1) <= now - _FLOOR_MIN_AGE) - - -def _sqp_depth_observation(request, asin, period, start, now): - cx = _client() - end = _sqp_depth_shift(start, period, 1) - row = {"period": period, "start": start.date().isoformat(), - "end": (end - timedelta(days=1)).date().isoformat(), "reportId": None, - "state": "empty", "hasRows": False, "floorEligible": False} - try: - text, meta = request(cx.REPORT_TYPES[SQP_TABLE]["report"], start=_iso(start), - end=_iso(end - timedelta(seconds=1)), - options={"reportPeriod": period, "asin": asin}) - except cx.AmazonError as exc: - words, status = str(getattr(exc, "vendor_reason", "") or exc), getattr(exc, "status", None) - if status not in ("FATAL", 400) or not _AMAZON_FLOOR_TEXT.search(words): - raise - row.update(state="out_of_range", amazonStatus=status, amazon=words, - reportId=getattr(exc, "report_id", None)) - row["floorEligible"] = _sqp_depth_eligible(row, now) - return row - _columns, _rows, _totals, count = _sqp_document_rows( - text, cx.REPORT_TYPES[SQP_TABLE], [asin], row, meta) - row.update(state="served" if count else "empty", hasRows=bool(count), - reportId=meta.get("reportId"), reportDocumentId=meta.get("reportDocumentId"), amazonStatus="DONE") - return row - - -def _sqp_depth_linear(rows, start, period): - refusals = 0 - while start.date().isoformat() in rows: - row = rows[start.date().isoformat()] - refusals = refusals + 1 if row["floorEligible"] else 0 - if refusals >= 2: - return None - start = _sqp_depth_shift(start, period, -1) - return start - - -def _sqp_depth_next(rows, newest, period): - served = sorted(day for day, row in rows.items() if row["state"] == "served") - if not served: - return _sqp_depth_linear(rows, newest, period) - first = _sqp_depth_start(served[0], period) - refused = sorted(day for day, row in rows.items() if row["floorEligible"] and day < served[0]) - if refused: - low = _sqp_depth_start(refused[-1], period) - distance = ((first - low).days // 7 if period == "WEEK" else - (first.year - low.year) * 12 + first.month - low.month) - if distance > 1: - middle = _sqp_depth_shift(low, period, distance // 2) - if middle.date().isoformat() not in rows: - return middle - return _sqp_depth_linear(rows, _sqp_depth_shift(first, period, -1), period) - - -def _sqp_depth_result(rows, period, newest, reports, *, done=False): - served = sorted(day for day, row in rows.items() if row["state"] == "served") - first = _sqp_depth_start(served[0], period) if served else None - bounded = bool(first) and all(rows.get(_sqp_depth_shift(first, period, -n).date().isoformat(), {}).get( - "floorEligible", False) for n in (1, 2)) - return {"period": period, "newest": newest.date().isoformat(), - "earliestServed": served[0] if served else None, - "floor": served[0] if done and bounded else None, - "state": ("measured" if bounded else "unproven") if done else "running", - "reportsRequested": reports, "observations": [rows[day] for day in sorted(rows, reverse=True)]} - - -def sqp_depth_walk(request, asin, period, newest, known=(), *, now=None, checkpoint=None): - """Find one ASIN's served boundary, retaining empty periods and requiring two adjacent old refusals.""" - if not isinstance(asin, str) or not re.fullmatch(r"[A-Z0-9]{10}", asin): - raise ValueError("SQP depth needs one ASIN") - newest, now = _sqp_depth_start(newest, period), _utc(now) - if _sqp_depth_shift(newest, period, 1) > now: - raise ValueError("SQP depth needs a completed newest period") - rows = {} - for saved in known: - row = dict(saved) - start = _sqp_depth_start(row["start"], period) - if row.get("period") != period or start > newest or row.get("state") not in ( - "served", "empty", "out_of_range"): - raise ValueError("SQP depth known observations must belong to this period and window") - end = (_sqp_depth_shift(start, period, 1) - timedelta(days=1)).date().isoformat() - if row.get("end") != end or (row["state"] == "served" and row.get("hasRows") is not True): - raise ValueError("SQP depth known observations need matching windows and row evidence") - row["floorEligible"] = _sqp_depth_eligible(row, now) - rows[row["start"]] = row - reports = 0 - while (start := _sqp_depth_next(rows, newest, period)) is not None: - row = _sqp_depth_observation(request, asin, period, start, now) - rows[row["start"]] = row - reports += 1 - if checkpoint: - checkpoint(_sqp_depth_result(rows, period, newest, reports)) - result = _sqp_depth_result(rows, period, newest, reports, done=True) - if checkpoint: - checkpoint(result) - return result - - def _dated_table(table): """A table whose first load plans dated history windows.""" spec = _client().REPORT_TYPES.get(table) @@ -7046,23 +6793,20 @@ def _seller_floor(con, end): f"{DEFAULT_BACKFILL_DAYS} day fallback.") -def _history_limit_key(table, period=None): - """Keep the legacy MONTH key and isolate every other period's observed history floor.""" - suffix = "" if period in (None, "MONTH") else "_" + period - return entity_key(f"limit_{table}_history{suffix}") +def _history_limit_key(table): + """Amazon's observed floor for one table; `limit_` family, so a table reset clears it too.""" + return entity_key(f"limit_{table}_history") -def _observed_history_limit(con, table, period=None): +def _observed_history_limit(con, table): """-> (floor, reason) Amazon showed by refusing an older window, or None.""" - state = _read_state(con, _history_limit_key(table, period)) + state = _read_state(con, _history_limit_key(table)) try: detail = json.loads(state["detail"]) if state and state.get("detail") else {} if detail.get("floor"): return _utc(detail["floor"]), str(detail.get("reason") or "") except (ValueError, TypeError, AttributeError): pass - if period not in (None, "MONTH"): - return None legacy = _read_state(con, entity_key("limit_" + table)) try: reason = json.loads(legacy["detail"]).get("reason", "") if legacy and legacy.get("detail") else "" @@ -7074,13 +6818,7 @@ def _observed_history_limit(con, table, period=None): return None -def _history_rule(spec, period=None): - """A period-specific rule replaces the table default without inheriting another period's floor.""" - rule = spec.get("history") or {} - return rule.get("periods", {}).get(period, rule) - - -def history_bound(con, table, end, *, now=None, period=None): +def history_bound(con, table, end, *, now=None): """-> (floor, cause): the oldest history start for `table`, and why when Amazon or the fallback sets it. max(seller floor, the table's `history` rule, Amazon's observed limit), on the table's window grain. @@ -7088,9 +6826,9 @@ def history_bound(con, table, end, *, now=None, period=None): `end` is the frozen cohort end (seller floor); `now` is the clock a rolling Amazon rule moves with. """ spec = _client().REPORT_TYPES.get(table) or {} - grain = "week" if period == "WEEK" else _history_grain(spec) + grain = _history_grain(spec) now = _utc(now) if now is not None else datetime.now(timezone.utc) - rule = _history_rule(spec, period) + rule = spec.get("history") or {} if rule.get("from") or rule.get("seller_floor") is False: # Brand-level data (Brand Analytics) predates the seller's first order, so the seller month # does not bound it: a `from` rule is the floor, else history walks back until Amazon refuses. @@ -7112,36 +6850,29 @@ def history_bound(con, table, end, *, now=None, period=None): elif rule.get("from"): bound = _grain_ceil(_date_utc(rule["from"]), grain) rule_cause = f"Amazon serves this data back to {bound.date().isoformat()} only." - if rule.get("provisional"): - rule_cause = (f"Weekly history starts at the provisional planning bound {bound.date().isoformat()}. " - "The earliest served week is unproved; an older range refusal sets the observed floor.") if bound is not None and bound > floor: floor, cause = bound, rule_cause - observed = _observed_history_limit(con, table, period) + observed = _observed_history_limit(con, table) if observed and observed[0] > floor: floor, cause = observed[0], observed[1] or ( f"Amazon keeps this data back to {observed[0].date().isoformat()}; older history is not available.") - if (spec.get("weekly") and not WEEKLY_HISTORY_PLANNED - and os.environ.get("AIOS_AMAZON_WEEKLY_HISTORY") != "1"): + if spec.get("weekly") and not WEEKLY_HISTORY_PLANNED: latest = _week_start(_utc(end)) - timedelta(days=7) if latest > floor: older = f" Amazon serves weeks back to {floor.date().isoformat()}." if bound is not None else "" floor, cause = latest, ("The latest completed week loads first; older weeks are not loaded yet, " - "because weekly history requires AIOS_AMAZON_WEEKLY_HISTORY=1." + older) + "because the weekly sync defers a past week until its next weekly turn." + older) return floor, cause -def history_floor(con, table, end, *, now=None, period=None): +def history_floor(con, table, end, *, now=None): """The oldest history start `first_load_plan` and `needs_first_load` use for `table`.""" - return history_bound(con, table, end, now=now, period=period)[0] + return history_bound(con, table, end, now=now)[0] def history_starts(con, tables, end, *, now=None): """{table: history_floor} for each dated table; the one dict every first-load planner passes.""" - starts = {table: history_floor(con, table, end, now=now) for table in tables if _dated_table(table)} - if SQP_TABLE in starts and "WEEK" in _client().REPORT_TYPES[SQP_TABLE].get("periods", ()): - starts[SQP_TABLE, "WEEK"] = history_floor(con, SQP_TABLE, end, now=now, period="WEEK") - return starts + return {table: history_floor(con, table, end, now=now) for table in tables if _dated_table(table)} def history_status(con, tables, end): @@ -7167,7 +6898,8 @@ def _history_reopened(con, table, cohort): return False bound = history_floor(con, table, cohort) if table == SQP_TABLE: - return bool(_sqp_catalogue(con)) and not _sqp_history_done(con, cohort) + return bool(_sqp_catalogue(con)) and any( + not _sqp_month(con, w0, w1)["batches"] for w0, w1 in _month_windows(bound, cohort)) floor = read_backfill_floor(con, table) return floor is not None and floor > bound @@ -7468,12 +7200,11 @@ def _report_document(con, table, fields, report_type, **kwargs): return result -def _newer_history_landed(con, table, end, period=None): +def _newer_history_landed(con, table, end): """A window of `table` newer than `end` has landed, so a refusal at `end` is a floor, not an outage.""" if table == SQP_TABLE: - windows = _week_windows if period == "WEEK" else _month_windows return any(any(not batch.get("refused") for batch in _sqp_month(con, w0, w1)["batches"].values()) - for w0, w1 in windows(end, datetime.now(timezone.utc))) + for w0, w1 in _month_windows(end, datetime.now(timezone.utc))) floor = read_backfill_floor(con, table) return floor is not None and end <= floor @@ -7488,11 +7219,10 @@ def _close_history_floor(con, table, exc, start, end): return None now = datetime.now(timezone.utc) status = getattr(exc, "status", None) - period = _sqp_period(start, end) if table == SQP_TABLE else None words = str(getattr(exc, "vendor_reason", "") or exc) floor = read_backfill_floor(con, table) said = (bool(_AMAZON_FLOOR_TEXT.search(words)) and (status in ("FATAL", 400) or " answered 400" in str(exc)) - and end <= now - _FLOOR_MIN_AGE and _newer_history_landed(con, table, end, period)) + and end <= now - _FLOOR_MIN_AGE and _newer_history_landed(con, table, end)) unexplained = (table != SQP_TABLE and status == "FATAL" and not getattr(exc, "has_error_document", True) and floor is not None and end <= floor and start < now - timedelta(days=365) and bool(_read_state(con, entity_key("reconcile_" + table)))) @@ -7502,25 +7232,14 @@ def _close_history_floor(con, table, exc, start, end): reason = (f"Amazon keeps this report back to {stop.date().isoformat()}; older history is not available. " "Keep the connection enabled for newer data; ask Seller Support for older history.") amazon = " ".join(words.replace("\u2014", "; ").replace("\u2013", " to ").split())[:300] - _record_state(con, _history_limit_key(table, period), "done", detail=json.dumps({ - "floor": _iso(stop), "period": period, "reason": reason, "amazon": amazon, - "window": [_iso(start), _iso(end)]})) + _record_state(con, _history_limit_key(table), "done", detail=json.dumps({ + "floor": _iso(stop), "reason": reason, "amazon": amazon, "window": [_iso(start), _iso(end)]})) if table != SQP_TABLE: # SQP's limit_ key is its progress document; its floor lives in the history marker only. _record_state(con, entity_key("limit_" + table), "done", detail=json.dumps({"reason": reason})) return reason -def _history_task_period(task): - return _sqp_period(task[1], task[2]) if task[0] == SQP_TABLE and task[1] is not None else None - - -def _history_cut_tasks(tasks, table, period, end): - """Only the refused period's windows at or below its floor are no longer owed.""" - return (task for task in tasks if task[0] == table and task[2] is not None - and task[2] <= end and _history_task_period(task) == period) - - def _record_reconciliation(con, table, marker): """Keep the most recent landed period, even while backfill travels backwards.""" key = entity_key("reconcile_" + table) @@ -9146,156 +8865,6 @@ def _order_metric_totals(records, *, flattened=False): return totals -ORDER_LINE_PROOF = "reconcile_order_items_" - - -def _order_report_months(start, end): - first = _utc(start).replace(day=1, hour=0, minute=0, second=0, microsecond=0) - while first < _utc(end): - following = (first.replace(day=28) + timedelta(days=4)).replace(day=1) - yield first, following - first = following - - -def _order_previous_months(con, orders): - if not {"amazon_order_id", "purchase_date"} <= _columns_of(con, "amzn_order_items"): - return set() - result = con.execute("SELECT DISTINCT strftime(TRY_CAST(purchase_date AS TIMESTAMPTZ), '%Y-%m-01') " - "FROM amzn_order_items WHERE amazon_order_id IN (SELECT unnest(?))", [list(orders)]) - return {month for (month,) in result.fetchall()} - - -def _order_line_summary(summary, row): - status = str(row.get("order_status") or "").strip().lower() - if status in ("cancelled", "canceled"): - return - summary["count"] += 1 - origin = f"{row['_report_document_id']}:{row['_report_row_number']}".encode() - summary["origin_xor"] = str(int(summary["origin_xor"]) ^ int.from_bytes( - hashlib.md5(origin, usedforsecurity=False).digest(), "little")) - currency = row.get("currency") - if not status or not currency or (summary["currency"] and summary["currency"] != currency): - summary["invalid"] = True - summary["currency"] = currency - try: - price = Decimal(str(row.get("item_price"))) - if not price.is_finite() or price < 0: - raise ValueError("invalid order line price") - summary["sales"] = str(Decimal(summary["sales"]) + price) - except (InvalidOperation, ValueError): - summary["invalid"] = True - - -def _order_report_observations(con, cols, rows, meta, report_type, start, end): - """Measure raw report windows before order grouping, typing or insertion.""" - spec = _client().REPORT_TYPES["order_items"] - required = {"amazon_order_id", "purchase_date", "asin", "order_status", "item_price", "currency"} - proof = {"reportType": report_type, "reportDocumentId": meta.get("reportDocumentId"), - "start": _iso(start), "end": _iso(end), "months": {}, "touched": set()} - if report_type == spec["report"]: - for first, following in _order_report_months(start, end): - proof["months"][first.date().isoformat()] = { - "start": _iso(max(first, start)), "end": _iso(min(following, end)), "asins": {}, - "invalid": bool(cols and not required <= set(cols))} - orders = set() - for row in rows: - orders.add(str(row.get("amazon_order_id") or "")) - if len(orders) >= 1000: - proof["touched"].update(_order_previous_months(con, orders)) - orders.clear() - try: - purchased = _utc(row["purchase_date"]) - if not row["purchase_date"]: - raise ValueError("missing purchase date") - except (KeyError, TypeError, ValueError, AttributeError): - proof["touched"].add(None) - continue - month = purchased.strftime("%Y-%m-01") - proof["touched"].add(month) - if report_type != spec["report"] or not start <= purchased < end: - continue - window = proof["months"][month] - if not row.get("asin") or not row.get("amazon_order_id"): - window["invalid"] = True - summary = window["asins"].setdefault(str(row.get("asin") or ""), - {"count": 0, "sales": "0", "currency": None, "invalid": False, "origin_xor": "0"}) - _order_line_summary(summary, row) - proof["touched"].update(_order_previous_months(con, orders)) - return proof - - -def _order_report_totals(windows): - totals = {} - for window in windows: - for asin, raw in window["asins"].items(): - total = totals.setdefault(asin, {"count": 0, "sales": "0", "currency": None, - "invalid": False, "origin_xor": "0"}) - total["count"] += raw["count"] - total["sales"] = str(Decimal(total["sales"]) + Decimal(raw["sales"])) - total["origin_xor"] = str(int(total["origin_xor"]) ^ int(raw["origin_xor"])) - total["invalid"] |= raw["invalid"] or bool( - raw["count"] and total["currency"] and total["currency"] != raw["currency"]) - if raw["count"]: - total["currency"] = raw["currency"] - return totals - - -def _record_order_report(con, proof): - """Publish raw line totals with disjoint purchase-date windows or an explicit invalidation.""" - prefix = entity_key(ORDER_LINE_PROOF) - touched = proof["touched"] | proof["months"].keys() - unknown = None in touched - if unknown: - touched.update(row[0][len(prefix):].replace("_", "-") + "-01" for row in con.execute( - "SELECT entity FROM _sync_state WHERE starts_with(entity, ?)", [prefix]).fetchall()) - for month in sorted(touched - {None}): - key = prefix + month[:7].replace("-", "_") - saved = _read_state(con, key) or {} - try: - previous = json.loads(saved.get("detail") or "{}") - windows = previous.get("windows", []) - if not isinstance(windows, list): - raise ValueError("invalid order source windows") - window = proof["months"].get(month) - windows = [old for old in windows if window and - (old["end"] <= window["start"] or old["start"] >= window["end"])] - except (ValueError, TypeError, KeyError, AttributeError): - windows, window = [], proof["months"].get(month) - if window and not unknown: - windows.append({**window, "reportDocumentId": proof["reportDocumentId"]}) - first = _utc(month + "T00:00:00Z") - following = (first.replace(day=28) + timedelta(days=4)).replace(day=1) - marker = {"version": 1, "source": _client().REPORT_TYPES["order_items"]["validates_against"], - "reportType": _client().REPORT_TYPES["order_items"]["report"], "cancelled_excluded": True, - "start": month, "end": following.date().isoformat(), "windows": windows, - "asins": _order_report_totals(windows), - "reportDocumentIds": list(dict.fromkeys(w["reportDocumentId"] for w in windows))} - if not windows: - marker["reason"] = "Order updates require purchase-month report proof to be refreshed" - marker["invalidated_by"] = {key: proof[key] for key in - ("reportType", "reportDocumentId", "start", "end")} - _record_state(con, key, "done", detail=json.dumps(marker)) - - -def _reset_order_report(con): - if _columns_of(con, "amzn_order_items"): - con.execute("DELETE FROM amzn_order_items") - con.execute("DELETE FROM _sync_state WHERE starts_with(entity, ?)", [entity_key(ORDER_LINE_PROOF)]) - - -def _write_order_report_window(con, cols, types, rows, proof, *, reset=False): - before = _reset_order_report if reset else None - after = lambda target: _record_order_report(target, proof) - if not cols: - with _report_transaction(con): - if before: - before(con) - after(con) - return 0, None - return replace_window(con, "amzn_order_items", cols, types, rows, replace_keys_for("order_items"), - before=before, after=after) - - def _order_status_observations(con, rows, meta): """Retain status bounds for the current Sales window in the existing account sync state.""" saved = _read_state(con, entity_key("reconcile_order_metrics")) @@ -10200,7 +9769,7 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri catch_up = {} def first_load_served(): - if first_load_job_live(rt): + if first_load_job_holds(rt): return True try: # A first-load pass rewrites this record, so a recent stamp means a producer is running. @@ -10249,18 +9818,14 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri and (table in completed or (read_watermark(con, table) and table not in repairs))] due_loaded = [] for table in loaded: - history = table == "brand_analytics" and table in reopened - hold = _loaded_refresh_hold(con, table, end, slot_pass=slot_pass, history=history) + hold = _loaded_refresh_hold(con, table, end, slot_pass=slot_pass) if hold is None: - if not history: - due_loaded.append(table) + due_loaded.append(table) continue next_due, reason = hold report["tables"][table] = {"state": "deferred", "reason": reason, "nextDue": _iso(next_due), "current": current_table(table)} log(f"{table}: DEFERRED: {reason}") - if history: - want.remove(table) if due_loaded: # Refreshes use their own floors; the history partition retains its bounded budget. con.close() @@ -10292,13 +9857,12 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri or table in completed or table in FEEDBACK_TABLES or (table == "settlements" and not refresh_loaded)): continue - tick = _read_state(con, entity_key("sync_tick_" + table)) - reason = ("This table has no recorded sync yet. Its first load is still running." if not tick else - "This table's first load is still running. Its previous rows were kept.") - report["tables"][table] = _deferred_table(reason, prior_refusals.get(table)) + report["tables"][table] = {"state": "deferred", "reason": + "The first load is continuing in bounded passes on the resident sync thread."} # The planner waits for the next phase on this hold. The card's tick stays as it was. _record_state(con, entity_key(PHASE_HOLD_PREFIX + table), "deferred") - if table == SQP_TABLE and tick and tick.get("state") == "throttled": + tick = _read_state(con, entity_key("sync_tick_" + table)) if table == SQP_TABLE else None + if tick and tick.get("state") == "throttled": # The first load owns SQP now; a passed SQP deadline must not make it due every minute. _record_state(con, entity_key("sync_tick_" + table), "deferred", tick.get("rows") or 0, tick.get("detail")) want = [table for table in want if (read_watermark(con, table) @@ -10327,7 +9891,8 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri plan = sync_plan(ticks, want, now=end, slot_pass=slot_pass) want = plan["tables"] for table, reason in plan["deferred"].items(): - report["tables"][table] = _deferred_table(reason, prior_refusals.get(table), current=current_table(table)) + report["tables"][table] = {"state": "deferred", "reason": reason, + "current": current_table(table)} log(f"{table}: DEFERRED: {reason}") elif not first_load: # The vendor's report floor also applies to an operator's Sync now click. @@ -10339,7 +9904,8 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri reason = _interval_reason(table, _read_state(con, entity_key("sync_tick_" + table)), datetime.now(timezone.utc)) if reason: - report["tables"][table] = _deferred_table(reason, prior_refusals.get(table), current=current_table(table)) + report["tables"][table] = {"state": "deferred", "reason": reason, + "current": current_table(table)} want.remove(table) # Heads go first even when the caller listed lines first. Scheduled plans still select one. @@ -10367,10 +9933,8 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri if prepare_brand_analytics(con): changed_tables.add(table_name("brand_analytics")) - def write_window(table, cols, types, rows, after=None, order_proof=None, reset=False): - if table == "order_items": - count, note = _write_order_report_window(con, cols, types, rows, order_proof, reset=reset) - elif cx.REPORT_TYPES[table].get("weekly"): + def write_window(table, cols, types, rows, after=None): + if cx.REPORT_TYPES[table].get("weekly"): count, note = write_week(con, table, cols, types, rows, tenant_key, report, log) else: count, note = replace_window(con, table_name(table), cols, types, rows, @@ -10441,7 +10005,20 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri report["nextDueSeconds"] = min(report.get("nextDueSeconds") or SQP_RESUME_SECONDS, SQP_RESUME_SECONDS) def sqp_progress(): - sqp_limit["progress"] = _sqp_progress(con, sqp_months, sqp_weeks, sqp_asins, batches, end) + """Batches done of total per planned month, from the durable month markers.""" + rows = [] + for w0, w1 in sqp_months: + marker = _sqp_month(con, w0, w1) + rows.append({"month": w0.strftime("%Y-%m"), "total": len(batches), + "done": sum(batch in marker["batches"] for batch in batches)}) + done, total = sum(r["done"] for r in rows), sum(r["total"] for r in rows) + weekly = sum(not _sqp_done(con, w0, w1, batch) for table, w0, w1, batch in work + if table == SQP_TABLE and w0 is not None and _sqp_period(w0, w1) == "WEEK") + from harness.amazon_client import REPORT_SUSTAINED_SECONDS + sqp_limit["progress"] = { + "months": rows, "batchesDone": done, "batchesTotal": total, + "monthsDone": sum(r["done"] >= r["total"] for r in rows), "monthsTotal": len(rows), + "etaSeconds": int((total - done + weekly) * REPORT_SUSTAINED_SECONDS)} if rows and batches else None try: measure_end = _utc(load["detail"]) if first_load and load and load.get("detail") else end @@ -10458,15 +10035,16 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri plans = {table: sync_window(table, None, now=end, floor=starts.get(table)) for table in pending} undocumented = repairs.intersection(pending) floors = {table: None if table in undocumented else read_backfill_floor(con, table) for table in pending} - wanted = [table for table in pending if (table not in completed or table in exhausted) and (table == SQP_TABLE or not read_watermark(con, table) + wanted = [table for table in pending if (table not in completed or table in exhausted) and (not read_watermark(con, table) or table in undocumented or table in reopened or read_watermark(con, table) < plans[table]["end"])] planned = first_load_plan(wanted, end - timedelta(days=DEFAULT_BACKFILL_DAYS), end, sqp_asins=sqp_asins, starts=starts) - for entry in _sqp_owed_tasks(con, _sqp_after_cohort(planned, end, sqp_asins)): + for entry in _sqp_after_cohort(planned, end, sqp_asins): task = entry if len(entry) == 4 else (*entry, None) table, w0, w1, batch = task if table == SQP_TABLE: - work.append(task) + if not _sqp_done(con, w0, w1, batch): + work.append(task) elif _after_cohort(table, w0, end): # A month after the frozen end has its own marker: the backfill floor only moves backwards. if not _cohort_month_done(con, table, w0): @@ -10511,22 +10089,16 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri if not recent <= mark <= end: mark = recent plan = plans[table] = sync_window(table, mark, now=end, since=since, - floor=history_floor(con, table, end), intraday=_lookback_intraday(con, table, end, slot_pass=slot_pass)) if table == SQP_TABLE: - # Monthly history precedes every completed week's ASIN batches. - start = plan["start"] if since is not None else history_floor(con, table, end) - months = list(reversed(_month_windows(start, plan["end"]))) - tasks = [(table, w0, w1, batch) for w0, w1 in months for batch in batches] - weekly_tasks = _sqp_week_tasks( - _utc(since) if since is not None else history_floor(con, table, end, period="WEEK"), - end, batches) - owed = [*tasks, *weekly_tasks] - if since is None: - owed = _sqp_owed_tasks(con, owed) - work.extend(owed) - if not any(_sqp_period(w0, w1) == "MONTH" for _table, w0, w1, _batch in owed): - work.append((table, None, None, None)) + # Closed months land newest first, before the latest week's batches. + months = list(reversed(_month_windows(plan["start"], plan["end"]))) + tasks = [(table, w0, w1, batch) for w0, w1 in months for batch in batches + if since is not None or not _sqp_done(con, w0, w1, batch)] + weekly_tasks = [task for task in _sqp_week_tasks( + _utc(since) if since is not None else end - timedelta(days=7), end, batches) + if since is not None or not _sqp_done(con, task[1], task[2], task[3])] + work.extend([*tasks, *weekly_tasks] if tasks else [*weekly_tasks, (table, None, None, None)]) else: work.append((table, None, None, None)) @@ -10562,15 +10134,12 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri sqp_limit["weeklyStart"] = (_week_start(end) - timedelta(days=7)).date().isoformat() if old_limit.get("weeklyStart") == sqp_limit["weeklyStart"]: sqp_limit["weeklyCause"] = old_limit.get("weeklyCause", "") - plan = plans.get(SQP_TABLE) or sync_window(SQP_TABLE, read_watermark(con, SQP_TABLE), now=end, since=since, - floor=history_floor(con, SQP_TABLE, end)) + plan = plans.get(SQP_TABLE) or sync_window(SQP_TABLE, read_watermark(con, SQP_TABLE), now=end, since=since) # A first load also owes the months that closed after its frozen cohort end. sqp_months = sorted({*_month_windows(plan["start"], plan["end"]), *(_month_windows(plan["end"], datetime.now(timezone.utc)) if first_load else ()), *((w0, w1) for name, w0, w1, _batch in work if name == SQP_TABLE and w0 is not None and _sqp_period(w0, w1) == "MONTH")}) - sqp_weeks = (_week_windows(history_floor(con, SQP_TABLE, end, period="WEEK"), end) - if "WEEK" in cx.REPORT_TYPES[SQP_TABLE].get("periods", ()) else []) sqp_limit["periods"] = [w0.strftime("%Y-%m") for w0, _w1 in sqp_months] sqp_progress() sqp_limit["deferredReason"] = ( @@ -10578,12 +10147,11 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri if not sqp_asins else sqp_limit["partialMonthReason"] if not sqp_limit["periods"] else "") _record_state(con, SQP_LIMIT_KEY, "done", detail=json.dumps(sqp_limit)) - history_cut = {} # Each table and period owns its observed floor. + history_cut = {} # table -> Amazon's floor observed this pass; older tasks are already counted landed. for index, (table, task_start, task_end, task_batch) in enumerate(work): if (first_load or table == SQP_TABLE) and table in failed: continue - history_key = (table, _history_task_period((table, task_start, task_end, task_batch))) - if history_cut.get(history_key) is not None and task_end is not None and task_end <= history_cut[history_key]: + if history_cut.get(table) is not None and task_end is not None and task_end <= history_cut[table]: continue if ((first_load or table == SQP_TABLE) and isinstance(report_budget, int) and documents >= max(0, report_budget) and (task_end is not None or table == "settlements" @@ -10621,7 +10189,7 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri pending_reports.add(table) report["nextDueSeconds"] = min(report.get("nextDueSeconds") or 900, 900) _record_state(con, entity_key(table), "pending", 0, weekly_pending["reason"]) - except cx.AmazonError as exc: + except Exception as exc: if isinstance(exc, cx.AmazonThrottled): problem = _sync_problem(exc, table, exc.operation or "report_document", tenant_key, rt, report, log) report["tables"][table] = problem @@ -10632,34 +10200,16 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri report["nextDueSeconds"] = min(report.get("nextDueSeconds") or wait, wait) _record_state(con, entity_key(table), "throttled", detail=problem["reason"]) continue - limit_reason = _close_history_floor(con, table, exc, task_start, task_end) - if limit_reason: - history_cut[history_key] = task_end - landed.update(_history_cut_tasks(work, table, "WEEK", task_end)) - sqp_limit["weeklyCause"] = limit_reason - sqp_limit["errors"] = [] - sqp_progress() - _record_state(con, SQP_LIMIT_KEY, "done", detail=json.dumps(sqp_limit)) - log(limit_reason) - continue causes.append(str(exc)) - if not _AMAZON_FLOOR_TEXT.search(str(getattr(exc, "vendor_reason", "") or exc)): + if isinstance(exc, cx.AmazonError) and not isinstance(exc, (cx.AmazonReportPending, cx.AmazonThrottled)): marker = _sqp_month(con, task_start, task_end) marker["batches"][task_batch] = {"refused": True, "reason": str(exc), "asins": []} - marker.setdefault("refusedAsins", {}).update({a: str(exc) for a in task_batch.split()}) _record_state(con, _sqp_month_key(task_start, "WEEK"), "refused", detail=json.dumps(marker)) landed.add((table, task_start, task_end, task_batch)) - else: - failed.add(table) - report["tables"][table] = {"state": "deferred", "reason": str(exc)} - except ValueError as exc: - # Unreadable source content remains owed without creating an ASIN refusal or history floor. - causes.append(str(exc)) if causes and not weekly_cause_logged: sqp_limit["weeklyCause"] = "Amazon weekly Search Query Performance: " + "; ".join(dict.fromkeys(causes)) sqp_problem(sqp_limit["weeklyCause"]) weekly_cause_logged = True - sqp_progress() _record_state(con, SQP_LIMIT_KEY, "done", detail=json.dumps(sqp_limit)) if weekly_deferred: break @@ -10898,7 +10448,7 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri floors[table] = w0 stop = _note_history_window(con, table, w0, count) if stop: - history_cut[history_key] = w0 + history_cut[table] = w0 landed.update(task for task in work if task[0] == table and task[2] is not None and task[2] <= w0) log(stop) @@ -10984,32 +10534,28 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri declared = {} if cols and (weekly or "_period_start" in replace_keys_for(table)): cols, rows = stamp_week(cols, rows, w0, w1) - order_proof = (_order_report_observations(con, cols, rows, meta, rtype, w0, w1) - if "order_items" in (table, sibling) else None) if not cols: if first_load: # Even an empty recent window clears old rows exactly once. A weekly # table never clears: R18's retention keeps watched history. for empty_table in (table, sibling): - if empty_table and empty_table != "order_items" and floors[empty_table] is None and not weekly: + if empty_table and floors[empty_table] is None and not weekly: if con.execute("SELECT 1 FROM information_schema.tables WHERE table_name=?", [table_name(empty_table)]).fetchone(): con.execute(f"DELETE FROM {table_name(empty_table)} WHERE 1=1") - cleared = table != "order_items" - elif order_proof is None: + cleared = True + else: completed_windows.append(window) continue if sibling: sibling_mirror = table_name(sibling) sibling_types = _ensure(con, sibling_mirror, cols, rows, declared=declared, preserve_rows=True) if cols else {} - sibling_reset = not sibling_cleared and not sibling_plan["incremental"] - if cols and sibling_reset and sibling != "order_items": + if cols and not sibling_cleared and not sibling_plan["incremental"]: con.execute(f"DELETE FROM {sibling_mirror} WHERE 1=1") - if cols or order_proof is not None: - sibling_written += write_window(sibling, cols, sibling_types, rows, - order_proof=order_proof, reset=sibling_reset) sibling_cleared = True + if cols: + sibling_written += write_window(sibling, cols, sibling_types, rows) order_observations = None if table == "orders": rows = _dedupe_orders(rows) @@ -11026,17 +10572,13 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri preserve_rows=True) if cols else {} # Explicit windows and first loads replace the mirror; incremental passes keep history. # A weekly table replaces by its period identity instead (D-2675). - order_reset = not cleared and not plan["incremental"] - if order_reset and not weekly and table != "order_items": + if not cleared and not plan["incremental"] and not weekly: con.execute(f"DELETE FROM {mirror} WHERE 1=1") cleared = True - if cols or table == "order_items": + if cols: after = (lambda target: _record_state(target, entity_key("reconcile_orders"), "done", detail=json.dumps(order_observations))) if order_observations else None - written += write_window(table, cols, types, rows, after=after, - order_proof=order_proof, reset=order_reset) - if table == "order_items": - cleared = True + written += write_window(table, cols, types, rows, after=after) if _periodic(cx.REPORT_TYPES[table]): _record_reconciliation(con, table, {"start": _iso(w0), "end": _iso(w1), "count": len(rows)}) @@ -11052,7 +10594,7 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri floors[reached] = w0 stop = _note_history_window(con, table, w0, len(rows) if cols else 0) if stop: - history_cut[history_key] = w0 + history_cut[table] = w0 landed.update(task for task in work if task[0] == table and task[2] is not None and task[2] <= w0) log(stop) @@ -11217,14 +10759,14 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri report["tables"][table] = {"state": "deferred", "reason": str(exc)} break except Exception as e: # noqa: BLE001 - limit_reason = (_close_history_floor(con, table, e, task_start, task_end) - if first_load or table == SQP_TABLE else None) + limit_reason = _close_history_floor(con, table, e, task_start, task_end) if first_load else None if limit_reason and table == SQP_TABLE: - history_cut[history_key] = task_end - landed.update(_history_cut_tasks(work, table, "MONTH", task_end)) + # Only months at or below Amazon's floor stop; newer months and the weekly batches continue. + history_cut[table] = task_end + landed.update(task for task in work if task[0] == table and task[2] is not None + and task[2] <= task_end) if sqp_limit is not None: sqp_limit["errors"] = [] - sqp_progress() _record_state(con, SQP_LIMIT_KEY, "done", detail=json.dumps(sqp_limit)) report["tables"][table] = {"state": "deferred", "current": True, "reason": limit_reason} log(limit_reason) @@ -11318,20 +10860,15 @@ def _sync_candidate(tenant_key="royal-imports", tables=None, since=None, log=pri sqp_progress() _record_state(con, SQP_LIMIT_KEY, "done", detail=json.dumps(sqp_limit)) for table in pending if first_load else (): - # Landed periods publish with their markers while the remaining history stays owed. + # Landed months publish with their markers, as SQP batches do; the months still owed run next pass. months = sum(task[0] == table and task[2] is not None for task in landed) row = report["tables"].get(table) - if (table == SQP_TABLE or not _periodic(cx.REPORT_TYPES[table]) or not months + if (table == SQP_TABLE or not cx.REPORT_TYPES[table].get("monthly") or not months or (row is not None and not (isinstance(row, dict) and row.get("state") in {"deferred", "throttled", "pending"}))): continue - grain = "week" if cx.REPORT_TYPES[table].get("weekly") else "month" - row = row or {"reason": f"{months} {grain}{'' if months == 1 else 's'} landed. The remaining {grain}s continue next pass."} - count_key = "landedWeeks" if grain == "week" else "landedMonths" - row_count = (con.execute(f"SELECT COUNT(*) FROM {table_name(table)}").fetchone()[0] - if _columns_of(con, table_name(table)) else 0) - report["tables"][table] = {**row, "state": "deferred", "current": True, - count_key: months, "rows": row_count} + row = row or {"reason": f"{months} month{'' if months == 1 else 's'} landed. The remaining months continue next pass."} + report["tables"][table] = {**row, "state": "deferred", "current": True, "landedMonths": months} if first_load: refusals ={row["table"]: row for row in _refused_reports(con)} not_offered = {table for table, row in refusals.items() if row["state"] == "not_offered"} @@ -13453,11 +12990,6 @@ def _sqp_resume_legs(ck, runtime, asins): con.executemany("INSERT INTO amzn_listings VALUES (?)", [(a,) for a in asins]) if watermark: record_watermark(con, SQP_TABLE, _utc(watermark)) - # A fixture watermark's earlier months need the same per-ASIN evidence as a real completed load. - for w0, w1 in _month_windows(history_floor(con, SQP_TABLE, _utc(watermark)), _utc(watermark)): - marker = _sqp_month(con, w0, w1) - marker["asins"] = {a: {"count": 0, "purchases": 0} for a in asins} - _sqp_save_month(con, w0, marker) calls.clear() return con @@ -14100,7 +13632,8 @@ def group_a_selftest(log=print, check=None): def catch_up_pass(served): # served: None = a first load stalled since its frozen end, "lease" = a job holds the Amazon - # lease, "fresh" = a first-load pass wrote its record an hour ago. + # lease, "fresh" = a first-load pass wrote its record an hour ago, "worker" = the scheduled + # pass's own resident writer holds the Amazon lease, "job" = a generation-style holder does. con = duckdb.connect(":memory:") con.execute("CREATE TABLE _sync_state (entity VARCHAR PRIMARY KEY, phase VARCHAR, last_id BIGINT, " "cursor_wd VARCHAR, rows BIGINT, updated_at VARCHAR)") @@ -14134,8 +13667,10 @@ def group_a_selftest(log=print, check=None): record_watermark(account, "orders", cohort) _record_state(account, entity_key(FIRST_LOAD_END_KEY), "running", detail=_iso(cohort)) CatchUpClock.instant = now - if served == "lease": - acquire_amazon_lease(rt, "group-a-first-load-job") + holder = {"lease": FIRST_LOAD_HOLDER_PREFIX + "group-a", "worker": "local@host:amazon:abc", + "job": FIRST_LOAD_HOLDER_PREFIX + "1234abcd"}.get(served) + if holder: + acquire_amazon_lease(rt, holder) first = (sync(rt.key, tables=(table,), rt=rt, log=lambda *_: None, scheduled=True) .get("tables") or {}).get(table) or {} first_asked = list(asked) @@ -14161,6 +13696,14 @@ def group_a_selftest(log=print, check=None): and all(nc[0].get("state") == "deferred" and "first load" in nc[0].get("reason", "") and not nc[1] and nc[3][0] is None and nc[3][1] is None for nc in controls)) ck("catalog Parent ASIN map fixtures and red-first controls", catalog_parent_selftest(log, check) == 0) + worker = catch_up_pass("worker") + ck("a scheduled pass with no producer refreshes an owed table while its own writer holds the Amazon lease", + worker[0].get("state") not in ("deferred", "failed", "throttled") and worker[0].get("rows") == 2 + and len(worker[1]) == 2 and (worker[3][1] or {}).get("state") == "done") + job = catch_up_pass("job") + ck("NC: a first-load generation holding the Amazon lease still defers an owed table", + job[0].get("state") == "deferred" and "first load" in job[0].get("reason", "") + and not job[1] and job[3][0] is None and job[3][1] is None) return 1 if failures else 0 @@ -14880,10 +14423,82 @@ def selftest(log=print): ck("NC-AZ38 brand analytics weekly wait and two week history fixtures", weekly_wait_selftest(log) == 0) ck("NC-AZ39 catalog picture fill fixtures", catalog_picture_selftest(log) == 0) ck("NC-AZ40 monthly tables after a frozen cohort fixtures", monthly_after_cohort_selftest(log) == 0) + ck("NC-AZ41 staging report hold during live's daily pass fixtures", report_hold_selftest(log) == 0) log(f"\n{'GREEN' if not fails else 'RED'}: {checks - len(fails)}/{checks}") return 0 if not fails else 1 +def report_hold_selftest(log=print, check=None): + """D-17410 offline fixtures: staging's report draws wait for live's daily pass, and every leg goes red without the hold.""" + from contextlib import redirect_stdout + from unittest.mock import patch + module = sys.modules[__name__] + cx = _client() + failures = [] + + def ck(name, ok): + if check: + check("[report hold] " + name, bool(ok)) + else: + log(f" {'ok' if ok else 'FAIL'} report hold {name}") + if not ok: + failures.append(name) + + at = lambda hour, minute=0: datetime(2026, 11, 18, hour, minute, tzinfo=timezone.utc).timestamp() # noqa: E731 + fields = {"seller_id": "SELFTEST-SELLER"} + + def target(value): + env = {k: v for k, v in os.environ.items() if k != "AIOS_FIRST_LOAD_TARGET"} + if value is not None: + env["AIOS_FIRST_LOAD_TARGET"] = value + return patch.dict(os.environ, env, clear=True) + + def legs(): + out = {} + with patch.object(module, "_LIVE_HOLD_NOTICED", set()): + docs = _ProcessDocuments() + printed = io.StringIO() + with target("staging"), redirect_stdout(printed): + held = take_report_token(docs, "s", now=at(1, 30)) + again = take_report_token(docs, "s", now=at(1, 45)) + lines = [line for line in printed.getvalue().splitlines() if "staging holds" in line] + out["held"] = (held == (False, 5400.0) and again[0] is False + and REPORT_LEDGER_KEY not in docs.docs and len(lines) == 1) + with target("staging"), redirect_stdout(io.StringIO()): + granted = take_report_token(docs, "s", now=at(3)) + row = ((docs.docs.get(REPORT_LEDGER_KEY) or {}).get("sellers") or {}).get("s") or {} + out["after"] = granted == (True, 0.0) and row.get("hours") == {"2026-11-18T03": 1} and row.get("spent") == 1 + live, unset = _ProcessDocuments(), _ProcessDocuments() + with redirect_stdout(io.StringIO()): + with target("staging"): + staged = take_report_token(_ProcessDocuments(), "s", now=at(1, 30)) + with target("live"): + live_draw = take_report_token(live, "s", now=at(1, 30)) + with target(None): + unset_draw = take_report_token(unset, "s", now=at(1, 30)) + out["scoped"] = staged[0] is False and live_draw == (True, 0.0) and unset_draw == (True, 0.0) + raised = None + with target("staging"), redirect_stdout(io.StringIO()), patch.object(time, "time", return_value=at(1, 30)): + try: + _ledger_draw(_ProcessDocuments())(fields) + except cx.AmazonThrottled as exc: + raised = exc + out["raise"] = (raised is not None and "Staging holds" in str(raised) and "03:00 UTC" in str(raised) + and raised.retry_after == 5400.0 and raised.operation == "createReport") + return out + + real = legs() + with patch.object(module, "live_daily_hold_seconds", return_value=0.0): + removed = legs() + ck("staging at 01:30Z is refused for 5400 s, spends no token and prints one hold line", + real["held"] and not removed["held"]) + ck("staging at 03:00Z is granted and its hour bucket counts 1", real["after"] and not removed["after"]) + ck("live or an unset target at 01:30Z is granted while staging waits", real["scoped"] and not removed["scoped"]) + ck("the daily pass draw raises AmazonThrottled naming the hold, with the hold as retry_after", + real["raise"] and not removed["raise"]) + return 1 if failures else 0 + + def catalog_picture_selftest(log=print, check=None): """Offline fixtures: `amazon_pictures` itself, and the post-publish fill reading the latest catalog snapshot.""" from unittest.mock import patch diff --git a/platform/harness/datastore.py b/platform/harness/datastore.py index f4e8f89a4877ebe1d79a2ccebf34cafa9cd4e417..a0f53d9d4b07a53ef3d7d17f9638e9e6c61fc1b7 100644 --- a/platform/harness/datastore.py +++ b/platform/harness/datastore.py @@ -3732,7 +3732,6 @@ REPLACE_EACH_PASS = { "product_tag": {"domain": []}, } REPLACE_PAGE = 5000 -REPLACE_READ_ATTEMPTS = 3 _connector_spec.register("odoo", _connector_spec.tables_for("odoo"), mirror=_connector_spec.ConnectorMirrorSpec(tuple(ENTITIES))) @@ -4019,13 +4018,13 @@ def _base_domain(spec): return list(spec.get("domain") or ()) + ([("active", "in", [True, False])] if spec["archivable"] else []) -def sync_entity(key, max_batches=200, batch=BATCH, log=print, report=None): +def sync_entity(key, max_batches=200, batch=BATCH, log=print): """Publish a single requested entity through the same source boundary as a full pass.""" if source_paused(): return _sync_entity(key, max_batches=max_batches, batch=batch, log=log) candidate = getattr(_SYNC_TLS, "candidate", None) if candidate is not None: - phase, pulled = _sync_one(key, max_batches=max_batches, batch=batch, log=log, report=report) + phase, pulled = _sync_one(key, max_batches=max_batches, batch=batch, log=log) # A recurring refresh may report changed pages to a bespoke cursor, but a # tenant pass cannot claim completion until those private pages finish too. pending = _connection_pending(candidate["con"], key) @@ -4034,7 +4033,7 @@ def sync_entity(key, max_batches=200, batch=BATCH, log=print, report=None): return phase, pulled rt = _resident_odoo_runtime() if rt is None: - return _sync_one(key, max_batches=max_batches, batch=batch, log=log, report=report) + return _sync_one(key, max_batches=max_batches, batch=batch, log=log) report = sync_all(entities=[key], max_batches=max_batches, log=log, batch=batch) return report[key]["phase"], report[key]["pulled"] @@ -4184,17 +4183,16 @@ def _sync_entity(key, max_batches=200, batch=BATCH, log=print): con.close() -def _sync_one(key, max_batches=200, batch=BATCH, log=print, report=None): +def _sync_one(key, max_batches=200, batch=BATCH, log=print): """One entity's pass: the cursor sync, the full replace, or the cursor sync then the replace.""" _connection_model(ENTITIES[key]) plan = REPLACE_EACH_PASS.get(key) if plan is None or source_paused(): return _sync_entity(key, max_batches=max_batches, batch=batch, log=log) if plan.get("scope") != "internal": - return _replace_entity(key, log=log, report=report) + return _replace_entity(key, log=log) phase, pulled = _sync_entity(key, max_batches=max_batches, batch=batch, log=log) - replaced, changed = _replace_entity(key, log=log, report=report) - return (phase if replaced == "live" else replaced), pulled + changed + return phase, pulled + _replace_entity(key, log=log)[1] def _replace_scope(con, key, plan): @@ -4208,44 +4206,37 @@ def _replace_scope(con, key, plan): return "(" + " OR ".join(terms) + ") AND " + _connection_where() -def _read_replace_rows(key, spec, domain): - """Read uncapped pages again if neither surrounding count matches, with a fixed attempt bound.""" - fields = spec["fields"] + list(spec.get("m2m") or []) - for _ in range(REPLACE_READ_ATTEMPTS): - expected = int(O.get_odoo().search_count(spec["model"], domain)) - rows, last = [], 0 - while True: - recs = O.search_read(spec["model"], domain + [("id", ">", last)], ["id"] + fields, - limit=REPLACE_PAGE, order="id asc") - if any(field not in rec for rec in recs for field in fields): - raise RuntimeError(f"{key}: full read omitted a requested field. The mirror keeps its previous rows.") - rows += _flatten_records(recs, spec["fields"], spec.get("m2m") or (), - named_m2m=spec.get("named_m2m"), raw_m2m=spec.get("raw_m2m"), spec=spec) - # Only an empty page ends a read, since Odoo may return short pages before the end. - if not recs: - break - last = recs[-1]["id"] - if len(rows) == expected: - return rows - after = int(O.get_odoo().search_count(spec["model"], domain)) - if len(rows) == after: - return rows - return None +def _replace_entity(key, log=print): + """Read every row the plan's domain matches, then swap the mirror's scope for it in one transaction. - -def _replace_entity(key, log=print, report=None): - """Replace a complete read atomically, or hold a moving entity before changing rows or state.""" + Returns (phase, changed_rows). A failed or short read raises before any delete. + """ if source_paused(): _paused_notice(f"full read of {key}", log) return PAUSED_PHASE, 0 spec, plan = ENTITIES[key], REPLACE_EACH_PASS[key] - rows = _read_replace_rows(key, spec, list(plan["domain"])) - if rows is None: - if report is not None: - report["cause"] = {"classifier": "odoo_entity_moving", "entity": key, - "attempts": REPLACE_READ_ATTEMPTS} - log(f" {key}: Odoo changed during all {REPLACE_READ_ATTEMPTS} full reads. Previous rows are kept.") - return "backfill", 0 + domain = list(plan["domain"]) + fields = spec["fields"] + list(spec.get("m2m") or []) + expected = int(O.get_odoo().search_count(spec["model"], domain)) + rows, last = [], 0 + while True: + recs = O.search_read(spec["model"], domain + [("id", ">", last)], ["id"] + fields, + limit=REPLACE_PAGE, order="id asc") + if any(field not in rec for rec in recs for field in fields): + raise RuntimeError(f"{key}: full read omitted a requested field. The mirror keeps its previous rows.") + rows += _flatten_records(recs, spec["fields"], spec.get("m2m") or (), + named_m2m=spec.get("named_m2m"), raw_m2m=spec.get("raw_m2m"), spec=spec) + # Stop on an EMPTY page: a server-side page cap returns short pages before the end. + if not recs: + break + last = recs[-1]["id"] + # Rows moved during the read when the read matches a second count. Refuse only a read + # that disagrees with both counts, before any delete. + if len(rows) != expected: + after = int(O.get_odoo().search_count(spec["model"], domain)) + if len(rows) != after: + raise RuntimeError(f"{key}: full read returned {len(rows)} rows, counts were " + f"{expected} and {after}. The mirror keeps its previous rows.") con = connect() try: _ensure_table(con, key, spec) @@ -4337,7 +4328,6 @@ def sync_all(entities=None, max_batches=200, log=print, after_entity=None, batch with binding: for key in selected: t0 = time.time() - row = out.setdefault(key, {"phase": "live", "pulled": 0, "secs": 0}) marker = _connection_state_key(key + ".pass") previous = candidate["con"].execute("SELECT phase FROM _sync_state WHERE entity=?", [marker]).fetchone() if entry else None if entry and candidate.get("resuming") and previous == ("done",): @@ -4346,11 +4336,11 @@ def sync_all(entities=None, max_batches=200, log=print, after_entity=None, batch if entry: candidate["con"].execute("DELETE FROM _sync_state WHERE entity=?", [marker]) phase, pulled = sync_entity(key, max_batches=max_batches, log=log, - **({"batch": batch} if batch != BATCH else {}), - **({"report": row} if candidate is not None else {})) + **({"batch": batch} if batch != BATCH else {})) if entry and phase == "live": candidate["con"].execute("INSERT OR REPLACE INTO _sync_state VALUES (?,?,?,?,?,?)", [marker, "done", 0, None, 0, time.strftime("%Y-%m-%d %H:%M:%S")]) + row = out.setdefault(key, {"phase": "live", "pulled": 0, "secs": 0}) if phase != "live": row["phase"] = phase row["pulled"] += pulled diff --git a/platform/harness/mirror_store.py b/platform/harness/mirror_store.py index 0119483696b1e6cbccce7d4973869ea4b7e85be0..26ced67ab9742fa4f743be549afb2259e17ca574 100644 --- a/platform/harness/mirror_store.py +++ b/platform/harness/mirror_store.py @@ -355,8 +355,7 @@ def _sync_result(connector, report): completed = all(isinstance(row, dict) and row.get("phase") in {"live", "backfill"} for row in report.values()) return completed, completed and all(row["phase"] == "live" for row in report.values()) - if (report.get("problems") or report.get("audit_problems") or (connector == "amazon" - and any(row.get("state") == "refused" for row in (report.get("tables") or {}).values()))): + if report.get("problems") or report.get("audit_problems"): # A problem owned by one held table publishes the rest; it never makes the pass current. holds = for_connector(connector)._publication_holds(report, ()) if connector == "amazon" else None if holds is None or holds["blocked"]: @@ -381,34 +380,16 @@ def _sync_result(connector, report): return completed, current -def odoo_moving_entities(report): - """Return only typed, exhausted moving-entity holds whose cause matches their entity.""" - if not isinstance(report, dict): - return {} - return {name: row["cause"] for name, row in (report or {}).items() - if re.fullmatch(r"[a-z][a-z0-9_]{0,79}", str(name)) and isinstance(row, dict) - and row.get("phase") == "backfill" and isinstance(row.get("cause"), dict) - and row["cause"].get("classifier") == "odoo_entity_moving" - and row["cause"].get("entity") == name - and type(row["cause"].get("attempts")) is int and row["cause"]["attempts"] == 3} - - -def record_sync_attempt(connector, rt): - """Record writer activity without claiming a completed pass or replacing a previous refusal.""" - with _CATCH_UP_LOCK: - state = _catch_up_source(rt, connector) - if not state["current"]: - state.update(pass_pending=True, attempt_at=time.time()) - - def tracked_sync(connector, runtime, pull, *args, _catch_up_complete=True, **kwargs): """Count completed passes and require a clean result before clearing startup catch-up.""" connector = connector_spec.CONNECTOR_SPEC_KEYS.get(connector, connector) rt = runtime if rt is None: return pull(*args, **kwargs) - record_sync_attempt(connector, rt) with _CATCH_UP_LOCK: + state = _catch_up_source(rt, connector) + if not state["current"]: + state.update(pass_pending=True, attempt_at=time.time()) active = (connector, _tenant_key(rt.key)) _ACTIVE_PULLS[active] = _ACTIVE_PULLS.get(active, 0) + 1 try: @@ -540,99 +521,6 @@ def _sql_literal(value): return "'" + str(value).replace("'", "''") + "'" -def _merge_publication_entries(base, local, stored): - """Merge keyed observations against their common base, naming conflicting local changes.""" - missing = object() - merged, conflicts = {}, [] - for name in sorted(set(base) | set(local) | set(stored)): - before, ours, theirs = (entries.get(name, missing) for entries in (base, local, stored)) - if _same_publication_entry(ours, before): - value = theirs - elif _same_publication_entry(ours, theirs): - value = theirs - elif _same_publication_entry(theirs, before): - value = ours - else: - conflicts.append(name) - continue - if value is not missing: - merged[name] = value - return merged, conflicts - - -def _same_publication_entry(left, right): - """A content-addressed table retains its identity when only its write timestamp differs.""" - if isinstance(left, dict) and isinstance(right, dict): - return {k: v for k, v in left.items() if k != "written_at"} == { - k: v for k, v in right.items() if k != "written_at"} - return left == right - - -def _marker_freshness(row): - """An invalid observation time cannot outrank a valid source-state observation.""" - try: - return _utc(row[5]) - except (ValueError, OverflowError): - return datetime.min.replace(tzinfo=timezone.utc) - - -def _merge_publication_markers(base, local, stored): - """Merge source state independently of observation time, retaining its newest valid freshness.""" - versions = [{row[0]: tuple(row) for row in rows} for rows in (base, local, stored)] - states = [{name: row[:5] for name, row in version.items()} for version in versions] - merged, conflicts = _merge_publication_entries(*states) - for name, state in merged.items(): - matching = [version[name] for version in versions - if name in version and version[name][:5] == state] - merged[name] = max(matching, key=_marker_freshness) - return merged, conflicts - - -def _candidate_checkpoint(con): - """Read the private candidate's common-base manifest and authoritative local markers.""" - if not con.execute("SELECT 1 FROM information_schema.tables " - "WHERE table_name='_candidate_checkpoint'").fetchone(): - return None - row = con.execute("SELECT document FROM _candidate_checkpoint").fetchone() - return json.loads(row[0]) if row else None - - -def _candidate_base_tables(checkpoint): - """Keep initial local content provenance distinct from an actual saved manifest.""" - return {**checkpoint.get("local_tables", {}), **checkpoint["base"].get("tables", {})} - - -def _save_candidate_checkpoint(con, base, markers, *, local_tables=None): - """Keep per-table base provenance separate from the installed serving publication.""" - con.execute("CREATE TABLE IF NOT EXISTS _candidate_checkpoint (document VARCHAR)") - con.execute("DELETE FROM _candidate_checkpoint") - con.execute("INSERT INTO _candidate_checkpoint VALUES (?)", [json.dumps({ - "base": base, "markers": markers, "local_tables": local_tables or {}})]) - - -def _save_initial_candidate_checkpoint(store, rt, candidate, published): - """Capture pre-manifest local table content before private source work changes it.""" - base = datastore._publication(published, store.connector) - local_tables = {} - missing = store._tables(published) - set(base.get("tables", {})) - if missing: - sealed = store._sealed_mode(rt) - part_limit = store.part_max_bytes - (len(be.HEADER) + 28 if sealed else 0) - with tempfile.TemporaryDirectory(prefix="mirror-local-base-") as tmp: - for name in sorted(missing): - entry, _fresh = store._checkpoint_table(published, name, {}, set(), tmp, - sealed=sealed, part_limit=part_limit) - local_tables[name] = entry - _save_candidate_checkpoint(candidate, base, store._state_rows(published), local_tables=local_tables) - - -def _table_marker_rows(store, rows, tables): - """Pair held data markers and direct table status with their table's rows.""" - names = {row[0] for row in store._held_state_rows(rows, tables)} - names.update(row[0] for row in rows if row[0].split("__", 1)[-1] in tables) - return [row for row in rows if row[0] in names] - - class MirrorStore: """One connector policy, with state cached separately for each tenant mirror.""" @@ -1410,232 +1298,162 @@ class MirrorStore: return entry, fresh - def _rebase_publication(self, tenant_key, rt, con, base, manifest, rows, *, allowed, report): - """Keep stored changes and their markers, holding overlapping private changes for retry.""" - raw = self._download(rt, _MANIFEST_KEY) - if raw is None: - # A local snapshot identity alone does not prove a saved manifest existed. - if "tables" in base: - raise PublicationHeld(f"{self.label} checkpoint held: the saved base publication is missing.") - return rows, False - stored = json.loads(raw) - if stored.get("tenant") != _tenant_key(tenant_key): - raise ValueError("Mirror manifest belongs to another tenant") - has_base = con.execute("SELECT 1 FROM information_schema.tables " - "WHERE table_name='_candidate_base'").fetchone() - candidate_base = con.execute("SELECT identity FROM _candidate_base").fetchone() if has_base else None - base_id = candidate_base[0] if candidate_base else base.get("snapshot_id", "unknown") - if base_id != base.get("snapshot_id", "unknown"): - raise PublicationHeld(f"{self.label} checkpoint held: private rows have a different base publication.") - if (stored.get("snapshot_id", "unknown") == base_id - and stored.get("tables") == base.get("tables") - and stored.get("sync_state") == base.get("sync_state")): - return rows, False - checkpoint = _candidate_checkpoint(con) - tables, conflicts = _merge_publication_entries( - _candidate_base_tables(checkpoint) if checkpoint else base.get("tables", {}), - manifest["tables"], stored.get("tables", {})) - if not allowed: - names = ", ".join(conflicts) or "the source's held tables" - raise PublicationHeld(f"{self.label} checkpoint held: a newer stored publication changed {names}. " - "Private rows keep their previous base for the next pass.") - with tempfile.TemporaryDirectory(prefix="mirror-rebase-") as tmp: - self._before_restore(tenant_key, rt) - stored, stored_rows, _paths, _total, _refused = self._validated_snapshot( - tenant_key, rt, con, raw, tmp) - if checkpoint is None: - raise PublicationHeld(f"{self.label} checkpoint held: common-base markers are unavailable.") - base_rows = checkpoint["markers"] - markers, marker_conflicts = _merge_publication_markers(base_rows, rows, stored_rows) - versions = base_rows + rows + stored_rows - conflicts = sorted(set(conflicts) | { - name for name in report.get("held_tables", ()) - if set(marker_conflicts) & {row[0] for row in _table_marker_rows(self, versions, {name})}}) - held_markers = {row[0] for row in _table_marker_rows(self, versions, set(conflicts))} - if set(marker_conflicts) - held_markers: - raise PublicationHeld(f"{self.label} checkpoint held: a newer stored publication changed markers " - + ", ".join(marker_conflicts) + ". Private rows stay for the next pass.") - if conflicts: - report["conflicts"] = conflicts - report["problems"].append(f"{self.label} checkpoint held tables: a newer stored publication changed " - + ", ".join(conflicts) + ". Private rows keep their previous base for the next pass.") - tables.update({name: stored["tables"][name] for name in conflicts if name in stored["tables"]}) - markers = {name: row for name, row in markers.items() if name not in held_markers} - markers.update({row[0]: tuple(row) for row in stored_rows if row[0] in held_markers}) - manifest["tables"] = tables - return list(markers.values()), True - - - def _checkpoint_table(self, con, name, prior, named, tmp, *, sealed, part_limit): - """Prepare changed immutable content, reusing every part the base already names.""" - path = Path(tmp) / (name + ".parquet") - size = None - if "parts" not in prior: - con.execute(f"COPY (SELECT * FROM {_quoted_identifier(name)}) TO ? " - "(FORMAT PARQUET, COMPRESSION ZSTD)", [str(path)]) - size = path.stat().st_size - if size is None or size > part_limit: - return self._chunked_entry(con, name, Path(tmp), prior, named, sealed=sealed) - metadata = _blob_metadata(path.read_bytes()) - if (metadata["sha256"] == prior.get("sha256") and prior.get("key") - and (".sealed-v1." in prior["key"]) == sealed): - return None, [] - suffix = ".sealed-v1.parquet" if sealed else ".parquet" - key = _content_key(name, suffix, metadata) - metadata.update(rows=con.execute(f"SELECT count(*) FROM {_quoted_identifier(name)}").fetchone()[0], - written_at=_iso(datetime.now(timezone.utc)), key=key) - if key in named and named[key] != metadata["sha256"]: - raise ValueError(f"{self.label} mirror content key collides with a published blob") - return metadata, [] if key in named else [(key, path)] - - - def _checkpoint_tables(self, rt, con, manifest, dirty, named, tmp, *, sealed, part_limit): - """Prepare local changes and migrate legacy remote entries without dropping refused tables.""" - tables, changed, pending = self._tables(con), [], [] - legacy = {name for name, meta in manifest["tables"].items() - if (not meta.get("key") and "parts" not in meta) - or any((".sealed-v1." in key) != sealed for key, _ in self._entry_blob_keys(name, meta))} - for name in sorted(legacy - tables): - prior, migrated = manifest["tables"][name], [] - for index, (key, metadata) in enumerate(self._entry_blob_keys(name, prior)): - if (".sealed-v1." in key) == sealed: - migrated.append(dict(metadata)) - continue - path = Path(tmp) / f"{name}.migrate{index}.parquet" - path.write_bytes(self._checked_mirror_blob(rt, key, metadata)) - new_key = key.replace(".parquet", ".sealed-v1.parquet") - migrated.append({**metadata, "key": new_key}) - pending.append((new_key, path)) - manifest["tables"][name] = ({**prior, "parts": migrated} if "parts" in prior - else {**prior, **migrated[0]}) - changed.append(name) - for name in sorted((dirty | legacy | (tables - set(manifest["tables"]))) & tables): - entry, fresh = self._checkpoint_table(con, name, manifest["tables"].get(name, {}), named, tmp, - sealed=sealed, part_limit=part_limit) - if entry is not None: - manifest["tables"][name] = entry - changed.append(name) - pending.extend(fresh) - return changed, pending - - - def _upload_checkpoint_parts(self, tenant_key, rt, pending, part_limit, progress): - """Ledger immutable keys before uploading bounded parts, retaining byte progress on failure.""" - if any(path.stat().st_size > part_limit for _key, path in pending): - raise ValueError("mirror part exceeds the byte limit") - if pending: - self._ledger_add(rt, tenant_key, [key for key, _path in pending]) - for key, path in pending: - data = path.read_bytes() - self._upload(rt, key, data) - progress["uploaded"] += len(data) - - - def _publish_checkpoint(self, tenant_key, rt, con, cache, manifest, changed, pending, tmp, *, - publication, hold, rebase_allowed, report, log, progress): - """Publish one complete manifest after content upload and a fresh stored-base comparison.""" - checkpoint = _candidate_checkpoint(con) - old = checkpoint["base"] if checkpoint else cache["manifest"] - sealed = bool(manifest.get("blob_envelope_version")) - part_limit = self.part_max_bytes - (len(be.HEADER) + 28 if sealed else 0) - rows = self._state_rows(con) - if not self._state_complete(rows): - return None - suffix = ".sealed-v1" if sealed else "" - state_meta, _blobs = self._state_entry(rows, part_limit, suffix) - if not (publication or changed or state_meta != old.get("sync_state") - or manifest["tables"] != old.get("tables", {}) or old.get("version") != MANIFEST_VERSION): - return None - self._upload_checkpoint_parts(tenant_key, rt, pending, part_limit, progress) - rows, report["rebased"] = self._rebase_publication( - tenant_key, rt, con, old, manifest, rows, - allowed=rebase_allowed and publication is not None and hold is None, report=report) - state_meta, blobs = self._state_entry(rows, part_limit, suffix) - pending, named = [], self._manifest_keys(old) - for index, (key, state) in enumerate(blobs): - if key not in named: - path = Path(tmp) / f"sync_state.{index}.json" - path.write_bytes(state) - pending.append((key, path)) - self._upload_checkpoint_parts(tenant_key, rt, pending, part_limit, progress) - stamp = _iso(datetime.now(timezone.utc)) - manifest.update(sync_state=state_meta, writer=_writer(), written_at=stamp, **(publication or {})) - if publication is not None: - manifest["completed_at"] = stamp - if hold is not None: - manifest["hold"] = hold - self._upload(rt, _HELD_KEY, _mirror_json(manifest)) - return manifest - superseded = _superseded_hold(self, tenant_key, rt) - if superseded: - manifest["superseded_hold"] = superseded - published = self._upload(rt, _MANIFEST_KEY, _mirror_json(manifest)) - progress["uploaded"] += len(published) - if superseded is not None: - try: - rt.delete_path(self.prefix + _HELD_KEY) - except Exception as exc: - log(f"{self.label} superseded hold cleanup deferred ({type(exc).__name__}); " - "the next publication retries") - if publication is None: - datastore._write_publication(con, self.connector, manifest) - cache["manifest"] = manifest - cache["problems"].clear() - self._retire_blobs(tenant_key, rt, published, named, log, orphans=cache.get("state") != "refused") - return manifest - - def persist_mirror(self, tenant_key, rt, con, changed_tables, *, report, log=print, - _publication=None, _hold=None, _rebase_allowed=True): - """Checkpoint immutable parts and markers before publishing one manifest, retaining private work on failure.""" + _publication=None, _hold=None): + """Checkpoint changed tables and the connector's markers, publishing the manifest last. + + Candidates include partially written tables. Hashing their parquet locally avoids reuploading + identical content; no previous blobs are downloaded on a sync pass. Failed candidates remain + dirty for retry on the next pass, while landed local rows and cursors stay usable. + + S67-T32 (D-3481), the order that makes a killed checkpoint harmless: (1) every new blob gets a + CONTENT key, and a key the current manifest names is never written; (2) the key ledger lists + every new key before its upload; (3) the blobs upload; (4) ONE manifest upload, a single-row + upsert, publishes them; (5) only then are superseded blobs deleted. Dying anywhere before (4) + leaves the previous manifest naming only complete, untouched blobs. + """ candidate = getattr(datastore._SYNC_TLS, "candidate", None) if candidate is not None and _publication is None: if candidate["tenant"] != tenant_key or candidate["connector"] != self.connector: raise ValueError("Checkpoint belongs to another source candidate") - return - progress = {"uploaded": 0} + return # The enclosing source pass owns publication after every table completes. + uploaded = 0 with self.lock: cache = self.restore_mirror(tenant_key, rt, log=log) cache["dirty"].update(changed_tables) try: if cache.get("state") == "refused" and not self._refused_may_publish(tenant_key, rt, con, cache): - raise PublicationHeld( - f"{self.label} mirror checkpoint held: this process could not restore the saved snapshot " - "at start. The saved copy is kept; restore tenant-store access and restart to " - "restore it before checkpointing.") - checkpoint = _candidate_checkpoint(con) - old = checkpoint["base"] if checkpoint else cache["manifest"] + message = (f"{self.label} mirror checkpoint held: this process could not restore the saved snapshot " + "at start. The saved copy is kept; restore tenant-store access and restart to " + "restore it before checkpointing.") + report["problems"].append(message) + log(message) + return + old = cache["manifest"] sealed = self._sealed_mode(rt) - if old.get("blob_envelope_version") and not sealed: - raise tenant_keys.TenantKeyLocked("A sealed mirror cannot publish without its tenant key.") + if old.get('blob_envelope_version') and not sealed: + raise tenant_keys.TenantKeyLocked('A sealed mirror cannot publish without its tenant key.') + suffix = '.sealed-v1' if sealed else '' + part_limit = self.part_max_bytes - (len(be.HEADER) + 28 if sealed else 0) if old.get("snapshot_id") and _publication is None: raise ValueError("An identified source requires the complete sync publication boundary") - tables = self._tables(con) | set(cache.get("refused_tables", ())) + named = self._manifest_keys(old) + tables = self._tables(con) + refused_tables = set(cache.get("refused_tables", ())) manifest = {"version": MANIFEST_VERSION, "tenant": _tenant_key(tenant_key), - "tables": {name: meta for name, meta in old.get("tables", {}).items() if name in tables}} + "tables": {name: meta for name, meta in old.get("tables", {}).items() + if name in tables or name in refused_tables}} if sealed: - manifest["blob_envelope_version"] = be.VERSION - part_limit = self.part_max_bytes - (len(be.HEADER) + 28 if sealed else 0) + manifest['blob_envelope_version'] = be.VERSION + changed, pending = [], [] with tempfile.TemporaryDirectory(prefix="mirror-persist-") as tmp: - changed, pending = self._checkpoint_tables( - rt, con, manifest, cache["dirty"], self._manifest_keys(old), tmp, - sealed=sealed, part_limit=part_limit) - saved = self._publish_checkpoint(tenant_key, rt, con, cache, manifest, changed, pending, tmp, - publication=_publication, hold=_hold, rebase_allowed=_rebase_allowed, - report=report, log=log, progress=progress) - if _hold is not None: - return saved - if saved or not changed: + # A legacy entry (no content key) is a candidate too: the first checkpoint migrates. + legacy = {name for name, meta in manifest["tables"].items() + if (not meta.get("key") and "parts" not in meta) + or any((".sealed-v1." in key) != sealed + for key, _ in self._entry_blob_keys(name, meta))} + # Refused local tables still name good remote parts: migrate those bytes too. + for name in sorted(legacy - tables): + prior = manifest['tables'][name] + migrated = [] + for index, (key, metadata) in enumerate(self._entry_blob_keys(name, prior)): + if ('.sealed-v1.' in key) == sealed: + migrated.append(dict(metadata)) + continue + data = self._checked_mirror_blob(rt, key, metadata) + path = Path(tmp) / f'{name}.migrate{index}.parquet' + path.write_bytes(data) + new_key = key.replace('.parquet', '.sealed-v1.parquet') + migrated.append({**metadata, 'key': new_key}) + pending.append((new_key, path)) + manifest['tables'][name] = ({**prior, 'parts': migrated} if 'parts' in prior + else {**prior, **migrated[0]}) + changed.append(name) + for name in sorted((cache["dirty"] | legacy | (tables - set(manifest["tables"]))) & tables): + if not self._valid_table(name): + raise ValueError(f"{self.label} mirror table name is invalid") + prior = manifest["tables"].get(name, {}) + path = Path(tmp) / (name + ".parquet") + size = None + if "parts" not in prior: + con.execute(f"COPY (SELECT * FROM {_quoted_identifier(name)}) TO ? " + "(FORMAT PARQUET, COMPRESSION ZSTD)", [str(path)]) + size = path.stat().st_size + if size is None or size > part_limit: + entry, fresh = self._chunked_entry(con, name, Path(tmp), prior, named, sealed=sealed) + if entry is None: + continue + manifest["tables"][name] = entry + changed.append(name) + pending.extend(fresh) + continue + data = path.read_bytes() + metadata = _blob_metadata(data) + if (metadata["sha256"] == prior.get("sha256") and prior.get("key") + and (".sealed-v1." in prior["key"]) == sealed): + continue + key = _content_key(name, suffix + ".parquet", metadata) + metadata.update(rows=con.execute(f"SELECT count(*) FROM {_quoted_identifier(name)}").fetchone()[0], + written_at=_iso(datetime.now(timezone.utc)), key=key) + manifest["tables"][name] = metadata + changed.append(name) + if key in named: + # The same content under its own name is already stored and published. + if named[key] != metadata["sha256"]: + raise ValueError(f"{self.label} mirror content key collides with a published blob") + continue + pending.append((key, path)) + rows = self._state_rows(con) + state_meta, state_blobs = self._state_entry(rows, part_limit, suffix) + published = None + if self._state_complete(rows) and (_publication or changed or state_meta != old.get("sync_state") + or manifest["tables"] != old.get("tables", {}) + or old.get("version") != MANIFEST_VERSION): + for index, (state_key, state) in enumerate(state_blobs): + if state_key not in named: + state_path = Path(tmp) / f"sync_state.{index}.json" + state_path.write_bytes(state) + pending.append((state_key, state_path)) + if any(path.stat().st_size > part_limit for _key, path in pending): + raise ValueError("mirror part exceeds the byte limit") + if pending: + self._ledger_add(rt, tenant_key, [key for key, _path in pending]) + for key, path in pending: + data = path.read_bytes() + self._upload(rt, key, data) + uploaded += len(data) + stamp = _iso(datetime.now(timezone.utc)) + manifest.update(sync_state=state_meta, writer=_writer(), written_at=stamp, + **(_publication or {})) + if _publication is not None: + manifest["completed_at"] = stamp + if _hold is not None: + manifest["hold"] = _hold + self._upload(rt, _HELD_KEY, _mirror_json(manifest)) + return manifest + superseded = _superseded_hold(self, tenant_key, rt) + if superseded: + manifest["superseded_hold"] = superseded + published = self._upload(rt, _MANIFEST_KEY, _mirror_json(manifest)) + uploaded += len(published) + if superseded is not None: + # A newer publication replaces a stored hold. Nobody has to release it. + try: + rt.delete_path(self.prefix + _HELD_KEY) + except Exception: # noqa: BLE001 + log(f"{self.label} superseded hold pointer stays until the next publication") + if _publication is None: + datastore._write_publication(con, self.connector, manifest) + cache["manifest"] = manifest + cache["problems"].clear() + if published: + self._retire_blobs(tenant_key, rt, published, named, log, orphans=cache.get("state") != "refused") + if published or not changed: cache["dirty"].clear() - log(f"{self.label} mirror persisted to the store: {len(changed)} changed tables, " - f"{progress['uploaded']:,} bytes uploaded") - return saved - except PublicationHeld as exc: - report["problems"].append(str(exc)) - log(str(exc)) + log(f"{self.label} mirror persisted to the store: {len(changed)} changed tables, {uploaded:,} bytes uploaded") + return manifest if published else None except Exception as exc: - message = (f"{self.label} mirror persistence failed ({type(exc).__name__}); " - f"{progress['uploaded']:,} bytes uploaded. " + message = (f"{self.label} mirror persistence failed ({type(exc).__name__}); {uploaded:,} bytes uploaded. " "Local rows were kept; restore tenant-store access and resync before restarting.") report["problems"].append(message) log(message) @@ -1820,20 +1638,8 @@ def _export_tables(store, con, directory): return paths -def _candidate_matches(store, candidate, published, previous): - """Resume a known same-tenant common base even after another publication was installed.""" - if not previous: - return False - installed = datastore._publication(published, store.connector) - checkpoint = _candidate_checkpoint(candidate) - if checkpoint is None: - return previous[0] == installed.get("snapshot_id", "unknown") - base = checkpoint["base"] - return previous[0] == base.get("snapshot_id", "unknown") and base.get("tenant") == installed.get("tenant") - - def _open_candidate(store, tenant_key, rt, published): - """Resume known common-base provenance or seed private rows from the serving copy.""" + """Resume private rows only while their base publication still matches the serving copy.""" path = Path(rt.datastore_path).with_suffix(f".{store.connector}.candidate.duckdb") candidate = datastore.apply_limits(datastore.duckdb.connect(str(path))) datastore._ensure_sync_state(candidate) @@ -1843,9 +1649,7 @@ def _open_candidate(store, tenant_key, rt, published): candidate.execute("CREATE TABLE IF NOT EXISTS _candidate_held (name VARCHAR PRIMARY KEY)") previous = candidate.execute("SELECT * FROM _candidate_base").fetchone() base = datastore._publication(published, store.connector).get("snapshot_id", "unknown") - if _candidate_matches(store, candidate, published, previous): - if _candidate_checkpoint(candidate) is None: - _save_initial_candidate_checkpoint(store, rt, candidate, published) + if previous and previous[0] == base: return candidate, previous[1] started = _iso(datetime.now(timezone.utc)) try: @@ -1860,7 +1664,6 @@ def _open_candidate(store, tenant_key, rt, published): candidate.execute("DELETE FROM _candidate_pending") candidate.execute("DELETE FROM _candidate_held") candidate.execute("INSERT INTO _candidate_base VALUES (?,?)", [base, started]) - _save_initial_candidate_checkpoint(store, rt, candidate, published) return candidate, started except BaseException: candidate.close() @@ -1880,7 +1683,7 @@ def pending_sync_entities(connector, tenant_key, rt): candidate = datastore.apply_limits(datastore.duckdb.connect(str(path))) try: base = candidate.execute("SELECT identity FROM _candidate_base").fetchone() - if not _candidate_matches(store, candidate, published, base): + if not base or base[0] != datastore._publication(published, store.connector).get("snapshot_id", "unknown"): return None pending = {row[0] for row in candidate.execute("SELECT name FROM _candidate_pending").fetchall()} if store.connector == "odoo": @@ -1898,13 +1701,6 @@ def _holds_rows(holds): return holds is not None and bool(holds["tables"] or holds.get("accounts")) -def _held_tables(holds): - """Retain whole private tables and their original bases even when only an account is held.""" - if holds is None: - return set() - return set(holds["tables"]).union(*(hold["tables"] for hold in (holds.get("accounts") or {}).values())) - - def _hold_previous(store, candidate, published, tables): """Give each held table its published rows and data markers, inside the candidate's open transaction.""" prior = store._tables(published) @@ -1925,36 +1721,6 @@ def _hold_previous(store, candidate, published, tables): candidate.executemany("INSERT INTO _sync_state VALUES (?,?,?,?,?,?)", previous) -def _retain_conflicting_candidate(store, candidate, published, manifest, conflicts): - """Refresh independent tables while retaining held private rows, markers and their original common bases.""" - checkpoint = _candidate_checkpoint(candidate) - tables = set(conflicts) - private = _table_marker_rows(store, store._state_rows(candidate), tables) - saved_rows = store._state_rows(published) - held_names = {row[0] for row in _table_marker_rows(store, checkpoint["markers"] + saved_rows, tables)} - base_rows = [row for row in saved_rows if row[0] not in held_names] - base_rows += [row for row in checkpoint["markers"] if row[0] in held_names] - base = {**manifest, "snapshot_id": checkpoint["base"].get("snapshot_id", "unknown"), - "tables": {name: entry for name, entry in manifest["tables"].items() if name not in tables}} - base["tables"].update({name: entry for name, entry in _candidate_base_tables(checkpoint).items() - if name in tables}) - candidate.execute("BEGIN TRANSACTION") - try: - _hold_previous(store, candidate, published, (store._tables(candidate) | store._tables(published)) - tables) - owned = store._state_rows(candidate) - if owned: - candidate.executemany("DELETE FROM _sync_state WHERE entity=?", [[row[0]] for row in owned]) - rows = [row for row in saved_rows if row[0] not in held_names] + private - if rows: - candidate.executemany("INSERT INTO _sync_state VALUES (?,?,?,?,?,?)", rows) - _save_candidate_checkpoint(candidate, base, base_rows) - candidate.executemany("INSERT OR IGNORE INTO _candidate_held VALUES (?)", [[name] for name in conflicts]) - candidate.execute("COMMIT") - except BaseException: - candidate.execute("ROLLBACK") - raise - - def _publication_line(store, report, done, holds, *, snapshot_id=None, why=None): """One log line: what a pass published and what it held, each with its reason.""" rows = report.get("tables") or {} @@ -2124,7 +1890,7 @@ def sync_mirror(connector, tenant_key, rt, *, report=None, log=print, before_pub prior_snapshot_id = datastore._publication(published, store.connector).get("snapshot_id", "unknown") metadata.pop("rows") metadata["completed_at"] = _iso(datetime.now(timezone.utc)) - result = {"problems": [], "held_tables": _held_tables(holds)} + result = {"problems": []} save_started = time.monotonic() with tempfile.TemporaryDirectory(prefix="mirror-publish-") as tmp: exporting = time.monotonic() @@ -2142,18 +1908,9 @@ def sync_mirror(connector, tenant_key, rt, *, report=None, log=print, before_pub report["problems"].extend(problems) return raise RuntimeError("; ".join(problems)) - if report is not None and "problems" in report: - report["problems"].extend(result["problems"]) - retained = set(result.get("conflicts", ())) | result["held_tables"] - kept = bool(retained) try: installing = time.monotonic() - rows = store._state_rows(candidate) - if result.get("rebased"): - store._before_restore(tenant_key, rt) - _saved, rows, paths, _total, _refused = store._validated_snapshot( - tenant_key, rt, published, _mirror_json(manifest), tmp) - store._install_snapshot(published, manifest, rows, paths) + store._install_snapshot(published, manifest, store._state_rows(candidate), paths) installed = True store._after_publish(tenant_key, manifest) except BaseException: @@ -2163,19 +1920,20 @@ def sync_mirror(connector, tenant_key, rt, *, report=None, log=print, before_pub note("install", installing) for listener in list(_PUBLISH_LISTENERS): try: - listener(store.connector, tenant_key, manifest, rows, paths) + listener(store.connector, tenant_key, manifest, store._state_rows(candidate), paths) except Exception as exc: # The snapshot is saved and installed here. A failed hand-off only delays readers. log(f"{store.label} mirror hand-off failed ({type(exc).__name__}: {exc})") if _holds_rows(holds): candidate.execute("ROLLBACK") - if retained: - _retain_conflicting_candidate(store, candidate, published, manifest, retained) - if odoo and holds is not None: + candidate.execute("DELETE FROM _candidate_base") + candidate.execute("INSERT INTO _candidate_base VALUES (?,?)", [manifest["snapshot_id"], started]) + if odoo: # D-11329: a held entity keeps its private progress but never narrows the next pass. names = [[name] for name in sorted(holds["held"])] candidate.executemany("DELETE FROM _candidate_pending WHERE name=?", names) candidate.executemany("INSERT OR IGNORE INTO _candidate_held VALUES (?)", names) + kept = True if holds is not None: log(_publication_line(store, report, done, holds, snapshot_id=manifest["snapshot_id"])) if odoo: diff --git a/platform/modules/amazon_profit.py b/platform/modules/amazon_profit.py index 59d6316871b54a6954a4ae6bbebafe8eeb97a348..b4eec1ea7621e45881b0ce3995aa653bc8d9d074 100644 --- a/platform/modules/amazon_profit.py +++ b/platform/modules/amazon_profit.py @@ -31,11 +31,9 @@ import re import shutil import tempfile import threading -from decimal import Decimal, ROUND_HALF_UP from pathlib import Path from modules import amazon_sku_metrics as M -from modules import amazon_part_dates as _part_dates import core.user_tables as _registry from core import module_tables from core import amazon_rivals as _rivals @@ -121,58 +119,14 @@ def _field(key, label, ftype, description, *, computed, **extra): return out -def _progress_fields(description, *, key="projects", label="Projects", empty_label="No projects"): +def _progress_fields(description): """Project progress and its completed-count label, written by the Projects doors.""" - return (_field(key, label, "pct", description, computed=True, - format={"display": "progress", "labelKey": key + "_label", "emptyLabel": empty_label}), - _field(key + "_label", label + (" completed" if key == "projects" else " label"), "text", - "Completed projects of the counted ones." if key == "projects" else "Open PDP workflow stage and progress.", + return (_field("projects", "Projects", "pct", description, computed=True, + format={"display": "progress", "labelKey": "projects_label", "emptyLabel": "No projects"}), + _field("projects_label", "Projects completed", "text", "Completed projects of the counted ones.", computed=True)) -HISTORY_FIELD = _field("history", "History", "text", "Open the row's recorded changes.", computed=True) -ASIN_APP_KEYS = ("asin", "projects", "pdp_update", "trend_score", "search_trend", "market_share_trend", - "click_share_trend", "exposure", "funnel_score", "pdp_last_updated", "title_last_updated", - "bullets_last_updated", "images_last_updated", "aplus_last_updated", "history") -SEARCH_TRENDS = (("search", "search_trend", "Search trend", "int"), - ("market_share", "market_share_trend", "Market share trend", "pct"), - ("click_share", "click_share_trend", "Click share trend", "pct")) -EXPOSURES = {"under": "Under-exposed", "match": "Matched", "over": "Over-exposed", - "few": "Few", "none": "Not seen"} -PDP_PART_DATE_KEYS = tuple(part + "_last_updated" for part in ("title", "bullets", "images", "aplus")) -PDP_CELL_KEYS = ("pdp_update", "pdp_update_label", "pdp_update_more") + PDP_PART_DATE_KEYS -_PDP_PROGRESS_FIELDS = _progress_fields("Progress of the earliest open PDP workflow.", key="pdp_update", - label="PDP update", empty_label="No open workflow") -_SIGNAL_FIELDS = ( - _field("trend_score", "Trend score", "int", "Search trend score from 0 to 100.", computed=True), - *(_field(key, label, "pct", "Change against the same reported months last year.", computed=True) - for _, key, label, _ in SEARCH_TRENDS), - _field("exposure", "Exposure", "select", "Sales or click share against impression share.", - computed=True, options=list(EXPOSURES.values())), - _field("funnel_score", "Funnel score", "int", "Search-weighted funnel score, with 5 at the market.", - computed=True, format={"decimals": 1}), -) -_ASIN_DATE_FIELDS = tuple(_field(key, label, "date", "Latest recorded listing or part change.", computed=True) - for key, label in zip(("pdp_last_updated",) + PDP_PART_DATE_KEYS, - ("PDP last updated", "Title last updated", "Bullets last updated", - "Images last updated", "A+ last updated"))) -_SIGNAL_COMPANIONS = tuple(field for kind, key, label, now_type in SEARCH_TRENDS for field in ( - _field(key + "_series", label + " series", "text", "Every reported historical month, with raw values, ASIN gaps and reasons.", computed=True), - _field(key + "_now", label + " now", now_type, "Searches in the newest complete store month." if kind == "search" - else "Share aggregated over the twelve-month calculation window.", computed=True), - _field(key + "_reason", label + " reason", "text", "Why the year comparison is unavailable.", computed=True), - _field(key + "_now_reason", label + " now reason", "text", "Why the current value is unavailable.", computed=True), -)) + tuple(_field(key + "_reason", label + " reason", "text", "Why this signal is unavailable.", computed=True) - for key, label in (("trend_score", "Trend score"), ("funnel_score", "Funnel score"), ("exposure", "Exposure"))) -_ASIN_COMPANIONS = (_PDP_PROGRESS_FIELDS[1:] - + (_field("pdp_update_more", "More PDP workflows", "int", "Other open PDP workflows.", computed=True),) - + _SIGNAL_COMPANIONS - + (_field("exposure_label", "Exposure detail", "text", "Exposure multiple and comparison base.", computed=True), - _field("pdp_last_updated_reason", "PDP date reason", "text", "Why Amazon's listing date is unavailable.", computed=True))) -_ASIN_SOURCE_KEYS = tuple(f["key"] for f in _SIGNAL_FIELDS + _SIGNAL_COMPANIONS) + ( - "exposure_label", "pdp_last_updated", "pdp_last_updated_reason") - - def _metric_fields(skip=()): """The sibling metric module's columns, as computed fields of this module.""" return tuple(_field(key, label, ftype, description, computed=True, **dict(extra or {})) @@ -226,7 +180,7 @@ PARENT_COMPETITOR_FIELDS = tuple(_field(k, label, t, d, computed=True, **dict(x) for k, label, t, d, x in _rivals.PARENT_COLUMNS) COMPETITOR_KEYS = _rivals.ROW_KEYS + _rivals.PARENT_KEYS _SKU_PROJECT_FIELDS = _progress_fields("Average progress of the projects on this SKU, its ASIN and its parent ASIN.") -COMPUTED_FIELDS = (_ECON_FIELDS[:1] + _SKU_PROJECT_FIELDS[:1] + (HISTORY_FIELD, PICTURE_FIELD) + _ECON_FIELDS[1:] + _metric_fields() +COMPUTED_FIELDS = (_ECON_FIELDS[:1] + _SKU_PROJECT_FIELDS[:1] + (PICTURE_FIELD,) + _ECON_FIELDS[1:] + _metric_fields() + (HEALTH_NOTE_FIELD,) + _SKU_PROJECT_FIELDS[1:] + COMPETITOR_FIELDS) #: Owned columns: people edit them and a refresh never writes them. @@ -378,7 +332,7 @@ def cost_note(row, *, code=None, found=True): if source and row.get("cogs_per_unit") is None: return source if source.endswith(".") else source + "." return NO_COST_NOTE -_IDENTITY_KEYS = ("sku", "picture", "asin", "parent_asin", "title", "projects", "projects_label", "history") +_IDENTITY_KEYS = ("sku", "picture", "asin", "parent_asin", "title", "projects", "projects_label") _PERIOD_KEYS = tuple(k for k in COMPUTED_KEYS if k not in _IDENTITY_KEYS + COMPETITOR_KEYS) #: The profit block a rollup table carries; `_ECON_ADDITIVE` sums, margin and band recompute. @@ -397,12 +351,9 @@ def _grain_fields(key_column, grain): else (metric["parent_asin"], PICTURE_FIELD)) where = "this ASIN or its parent ASIN" if key_column == "asin" else "this parent ASIN" progress = _progress_fields(f"Average progress of the projects on {where}.") - app = (_PDP_PROGRESS_FIELDS[:1] + _SIGNAL_FIELDS + _ASIN_DATE_FIELDS + (HISTORY_FIELD,) - if key_column == "asin" else (HISTORY_FIELD,)) - companions = _ASIN_COMPANIONS if key_column == "asin" else () - return (ids[:1] + progress[:1] + app + ids[1:] + (econ["title"],) + rollup + tuple(econ[k] for k in _ROLLUP_ECON) + return (ids[:1] + progress[:1] + ids[1:] + (econ["title"],) + rollup + tuple(econ[k] for k in _ROLLUP_ECON) + _metric_fields(skip=("parent_asin",)) + (HEALTH_NOTE_FIELD,) - + progress[1:] + companions) + + progress[1:]) ASIN_FIELDS = _grain_fields("asin", "asin") @@ -448,7 +399,7 @@ PROJECT_COMPUTED_KEYS = tuple(f["key"] for f in PROJECT_COMPUTED_FIELDS) def table_specs(): """Every module table in creation order.""" - specs = ( + return ( {"key": TABLE_KEY, "label": TABLE_LABEL, "fields": COMPUTED_FIELDS + OWNED_FIELDS + TRACK_FIELDS, "owned": OWNED_KEYS, "keyColumn": "sku", "template": TEMPLATE_KEY}, {"key": ASIN_KEY, "label": ASIN_LABEL, "fields": ASIN_FIELDS + COMPETITOR_FIELDS, "owned": (), @@ -456,10 +407,6 @@ def table_specs(): {"key": PARENT_KEY, "label": PARENT_LABEL, "fields": PARENT_FIELDS + PARENT_COMPETITOR_FIELDS, "owned": (), "keyColumn": "parent_asin", "template": PARENT_TEMPLATE_KEY}, ) - for spec in specs: - fields = {f["key"]: f for f in spec["fields"]} - spec["fields"] = tuple(fields[key] for key in _project_field_order(fields)) - return specs # ---------------------------------------------------------------- period and threshold @@ -1760,7 +1707,7 @@ def _sync_cursor(tenant_key, rt): else: import duckdb cur = duckdb.connect(str(path), read_only=True) - if _mirror_has_economics(cur) or {'sku', 'attributes', 'summaries'} <= _columns(cur, 'amzn_listing_items'): + if _mirror_has_economics(cur): return cur, cur.close cur.close() for key in [k for k in _SNAPSHOT if k[0] == tenant_key]: @@ -1773,7 +1720,7 @@ def sync_matches(tenant_key, synced_tables, *, st=None, con=None, catalogue=None """Refresh the current period, apply decisions, then suggest for every unlinked SKU. Returns the report. It raises on failure; `on_amazon_sync` is the never-raise wrapper.""" synced = {str(t or "").strip().lower() for t in synced_tables or ()} - if not synced & {"economics", "listings", "listing_items"}: + if not synced & {"economics", "listings"}: return {"state": "skipped", "synced": sorted(synced)} if st is None: from harness import amazon_store @@ -1785,17 +1732,6 @@ def sync_matches(tenant_key, synced_tables, *, st=None, con=None, catalogue=None con, close = _sync_cursor(tenant_key, st) try: report = {"state": "done"} - if synced & {"listings", "listing_items"}: - try: - report["partDates"] = _part_dates.observe(st, con, day=today or _today()) - report["partDateCells"] = paint_asin_part_dates(st) - except Exception as exc: # quality-ok: Date storage failure must not block profit matching. - report['partDatesError'] = f'{type(exc).__name__}: {exc}' - _log(f"{tenant_key}: listing part dates failed ({report['partDatesError']}).") - if not synced & {'economics', 'listings'}: - return dict(report, state='failed', reason=report['partDatesError']) - if not synced & {"economics", "listings"}: - return report try: refreshed = refresh(st, con, by=WRITER, today=today, tenant=tenant_key) report["refresh"] = {k: refreshed.get(k) for k in ("skus", "linked", "refusal")} @@ -1871,16 +1807,12 @@ def _definition_fields(fields=None, owned_keys=None): def _project_field_order(keys): - """Pin App fields in schema defaults and keep companions last, without changing saved layouts.""" + """Place Projects beside a product identity in schema defaults, never saved user layouts.""" keys = list(keys) - if not keys or keys[0] not in ("sku", "asin", "parent_asin"): - return keys - pinned = ASIN_APP_KEYS if keys[0] == "asin" else (keys[0], "projects", "history") - companions = (("projects_label",) + tuple(f["key"] for f in _ASIN_COMPANIONS) - if keys[0] == "asin" else ()) - return ([key for key in pinned if key in keys] - + [key for key in keys if key not in pinned + companions] - + [key for key in companions if key in keys]) + if keys and keys[0] in ("sku", "asin", "parent_asin") and "projects" in keys: + keys.remove("projects") + keys.insert(1, "projects") + return keys def _format_repairs(fields, wanted): @@ -1900,9 +1832,9 @@ def _grain_word_repairs(fields, wanted): continue if field["key"] == "grain" and target.get("options") and field.get("options") != target["options"]: out.setdefault("grain", {})["options"] = list(target["options"]) - if field["key"] in ("projects", "pdp_update") and (field.get("type") != target["type"] or "link" in field + if field["key"] == "projects" and (field.get("type") != target["type"] or "link" in field or (field.get("format") or {}) != (target.get("format") or {})): - out[field["key"]] = {"type": target["type"], "format": dict(target.get("format") or {}), "link": None} + out["projects"] = {"type": target["type"], "format": dict(target.get("format") or {}), "link": None} return out @@ -2093,9 +2025,6 @@ def write_rows(st, computed, *, table_key=TABLE_KEY, key_column="sku", label=TAB for rid, row in table_rows(st, table_key).items() if str(row.get(key_column) or "").strip() not in computed} wanted = {rid: values for rid, values in wanted.items() if values} - for rid, row in table_rows(st, table_key).items(): - if row.get("history") != "View": - wanted.setdefault(rid, {})["history"] = "View" if wanted: import core.user_tables as ut @@ -2639,7 +2568,7 @@ def grain_rows(rows, key, *, thin=THIN_DEFAULT, period_label="", fields=()): flat["cost_note"] = _grain_cost_note(flat, children) flat["period"] = period_label out[value] = {k: stored_cell(_FIELD_BY_KEY.get(k), flat.get(k)) for k in keys - if k not in ("projects", "projects_label") + PDP_CELL_KEYS + _ASIN_SOURCE_KEYS} + if k not in ("projects", "projects_label")} return out @@ -2871,117 +2800,8 @@ def _withhold_untied(by_grain, checks): def _grain_period_keys(fields, key_column): return tuple(f["key"] for f in fields - if f["key"] not in (key_column, "picture", "parent_asin", "title", "projects", "projects_label", "history") - + COMPETITOR_KEYS + PDP_CELL_KEYS + _ASIN_SOURCE_KEYS) - - -def _listing_dates(con): - """Read the newest Amazon lastUpdatedDate per ASIN from raw listing summaries.""" - if "summaries" not in _columns(con, "amzn_listing_items"): - return {} - rows = con.execute("""SELECT json_extract_string(s.value, '$.asin'), - CAST(MAX(TRY_CAST(json_extract_string(s.value, '$.lastUpdatedDate') AS TIMESTAMPTZ)) - AT TIME ZONE 'UTC' AS DATE) - FROM amzn_listing_items l, json_each(TRY_CAST(l.summaries AS JSON)) s GROUP BY 1""").fetchall() - return {asin: date.isoformat() for asin, date in rows if asin and date is not None} - - -def _exposure_detail(exposure): - """Format the recorded exposure multiple with the prototype's one-decimal rounding.""" - if not exposure or exposure["multiple"] is None: - return None - multiple = Decimal.from_float(float(exposure["multiple"])).quantize(Decimal("0.1"), rounding=ROUND_HALF_UP) - return f"{multiple}x {exposure['base']} share" - - -def _asin_signal_cells(signal, history): - """Store scalar cells and raw monthly bars for every reported historical month.""" - out = {key: stored_cell(_FIELD_BY_KEY[key], signal[key]) for key in ("trend_score", "funnel_score")} - out.update({key + "_reason": value for key, value in signal["reasons"].items() - if key in ("trend_score", "funnel_score", "exposure")}) - exposure = signal["exposure"] - out["exposure"] = EXPOSURES[exposure["kind"]] if exposure else None - out["exposure_label"] = _exposure_detail(exposure) - for kind, key, _label, _now_type in SEARCH_TRENDS: - trend = signal["trends"][kind] - points = {point["month"]: point for point in trend["series"]} - series = [points[month] for month in history] - latest = series[-1] if series else {"value": None, "reason": signal["note"] or "No search report"} - now = latest["value"] if kind == "search" else signal[kind] - out.update({key: None if trend["change"] is None else str(trend["change"] * 100), - key + "_series": json.dumps(series, separators=(",", ":")), - key + "_now": (stored_cell(_FIELD_BY_KEY[key + "_now"], now) if kind == "search" - else None if now is None else str(now * 100)), - key + "_reason": trend["reason"], - key + "_now_reason": (signal["note"] or latest["reason"] if kind == "search" - else signal["reasons"][kind])}) - return out - - -def paint_asin_signals(st, con, *, today): - """Refresh source signals and Amazon's date for every stored ASIN, independent of the profit period.""" - from modules import amazon_search_signals as signals - asins = [str(row["asin"]).strip() for row in table_rows(st, ASIN_KEY).values() if row.get("asin")] - result = signals.signals(con, asins, today=today) - dates = _listing_dates(con) - parts = _part_dates.cells(st, asins) - computed = {asin: dict(_asin_signal_cells(signal, result["history"]), **parts[asin], pdp_last_updated=dates.get(asin), - pdp_last_updated_reason=None if asin in dates else "No listing lastUpdatedDate reported") - for asin, signal in result["asins"].items()} - return write_rows(st, computed, table_key=ASIN_KEY, key_column="asin", label=ASIN_LABEL, period_keys=()) - - -def paint_asin_part_dates(st): - """Refresh recorded part dates without profit validation, search reports or Odoo matching.""" - asins = [str(row['asin']).strip() for row in table_rows(st, ASIN_KEY).values() if row.get('asin')] - if not asins: - return {'rows': 0} - return write_rows(st, _part_dates.cells(st, asins), table_key=ASIN_KEY, - key_column='asin', label=ASIN_LABEL, period_keys=()) - - -def asin_signal_checks(st, con, *, today=None): - """Tie stored signal cells to the signal reader's independently validated raw monthly inputs.""" - from modules import amazon_search_signals as signals - rows = {str(row["asin"]).strip(): row for row in table_rows(st, ASIN_KEY).values() if row.get("asin")} - result = signals.signals(con, rows, today=today) - verified = signals.validate(con, result, today=today) - differences = sum(any(rows[asin].get(key) != value for key, value in _asin_signal_cells(signal, result["history"]).items()) - for asin, signal in result["asins"].items()) - return [{"check": "amazon_profit: stored ASIN search signals tie to raw monthly reports", - "ok": verified["ok"] and not differences, "rows": len(rows), "differences": differences, - "reason": verified["reason"]}] - - -def listing_date_checks(st, con): - """Independently tie stored PDP dates to the newest raw summary timestamp per ASIN.""" - newest = {} - available = "summaries" in _columns(con, "amzn_listing_items") - if available: - cursor = con.execute("SELECT summaries FROM amzn_listing_items") - for batch in iter(lambda: cursor.fetchmany(2048), []): - for (raw,) in batch: - try: - summaries = json.loads(raw) if isinstance(raw, str) else raw - except (TypeError, ValueError): - continue - for summary in summaries if isinstance(summaries, list) else (): - if not isinstance(summary, dict) or not summary.get("asin"): - continue - try: - stamp = dt.datetime.fromisoformat(str(summary.get("lastUpdatedDate")).replace("Z", "+00:00")) - except ValueError: - continue - stamp = stamp.replace(tzinfo=stamp.tzinfo or dt.timezone.utc).astimezone(dt.timezone.utc) - asin = summary["asin"] - if asin not in newest or stamp > newest[asin]: - newest[asin] = stamp - rows = list(table_rows(st, ASIN_KEY).values()) - differences = sum(row.get("pdp_last_updated") != (newest[row["asin"]].date().isoformat() - if row.get("asin") in newest else None) for row in rows) - return [{"check": "amazon_profit: PDP last updated ties to newest listing lastUpdatedDate per ASIN", - "ok": available and not differences, "rows": len(rows), "differences": differences, - "reason": None if available else "Listing summaries are unavailable"}] + if f["key"] not in (key_column, "picture", "parent_asin", "title", "projects", "projects_label") + + COMPETITOR_KEYS) def _date(value): @@ -3322,7 +3142,6 @@ def refresh(st, con, *, period=None, thin=None, by="", cost_reader=None, today=N report["parent"] = write_rows(st, by_grain["parent_asin"], table_key=PARENT_KEY, key_column="parent_asin", label=PARENT_LABEL, period_keys=_grain_period_keys(PARENT_FIELDS, "parent_asin")) - report["asinSignals"] = paint_asin_signals(st, con, today=day) report["blankReasons"] = blank_check(stored_tables(st)) bands = {sku: values.get("band") for sku, values in computed.items()} report["owned"] = apply_owned(st, bands, today=day, owner=owner_label) @@ -3468,7 +3287,6 @@ def _mirror_has_economics(con): def snapshot_cursor(tenant=TENANT_ZERO, tables=(ECONOMICS, SETTLEMENTS, LISTINGS, "amzn_orders", - "amzn_listing_items", "amzn_search_query_performance", "amzn_ads_sp_advertised_products", "amzn_ads_sb_advertised_products", "amzn_ads_sd_advertised_products")): """An in-memory DuckDB over the stored mirror snapshot. Downloads only; writes nothing back.""" @@ -3630,10 +3448,10 @@ def stored_grain_checks(st, con): def validate(pre=None, month=None, con=None, *, cost_usage=None, cost_history=None, typed_rows=None, - typed_values=None, typed_period=None, st=None, today=None): + typed_values=None, typed_period=None, st=None): """Tie net proceeds components to independent Amazon sources within 2% over settled days. Receipt COGS ties to Odoo; typed COGS (`typed_rows`) is listed apart and ties to its stored values. - With `st`, stored grain figures, search signals and listing dates tie to their sources.""" + With `st`, the stored ASIN and parent figures tie per row to the mirror and every blank says why.""" month = month or last_complete_month() bounds = period_bounds(month) start, end = bounds["start"], bounds["end"] @@ -3693,8 +3511,6 @@ def validate(pre=None, month=None, con=None, *, cost_usage=None, cost_history=No checks.extend(typed_cogs_checks(con, typed_rows, typed_values, cost_usage, typed_period or bounds)) if st is not None: checks.extend(stored_grain_checks(st, con)) - checks.extend(listing_date_checks(st, con)) - checks.extend(asin_signal_checks(st, con, today=today)) return checks diff --git a/platform/modules/amazon_profit_account.py b/platform/modules/amazon_profit_account.py index 3db4617a8cb6a943296fbaf65f79adf36ad5bacf..d42a19ca1622ba1d3f7573adf878b1e17f56b321 100644 --- a/platform/modules/amazon_profit_account.py +++ b/platform/modules/amazon_profit_account.py @@ -555,124 +555,6 @@ def validate_pace(con, bounds, pace, elapsed, month_days, reason): "checks": checks, "sourceTables": [ECONOMICS, "_sync_state"]} -class _SourceRows: - def __init__(self, rows): - self.rows = rows - - def fetchone(self): - return self.rows[0] if self.rows else None - - def fetchall(self): - return list(self.rows) - - -def _settlement_source(sql): - """Parse distinct settlement labels and referenced order dates without losing source lines or null keys.""" - expressions = {"posted": ("posted_date", f"CAST({profit._moment_sql('s.posted_date')} AS DATE)")} - for name, field in (("description", "amount_description"), ("kind", "amount_type"), - ("transaction", "transaction_type")): - expressions[name] = field, f"lower(regexp_replace(COALESCE({field}, ''), '[^a-zA-Z0-9]', '', 'g'))" - fields = [field for field, _expression in expressions.values()] - projection = ", ".join(expression.replace("s.posted_date", "posted_date") + " AS " + name - for name, (_field, expression) in expressions.items()) - keys = ", ".join(f"{field} AS source_{field}" for field in fields) - labels = (f"labels AS MATERIALIZED (SELECT {keys}, {projection} " - f"FROM (SELECT DISTINCT {', '.join(fields)} FROM {profit.SETTLEMENTS})), ") - for name, (_field, expression) in expressions.items(): - sql = sql.replace(expression + " AS " + name, "label." + name + " AS " + name) - join = " AND ".join(f"s.{field} IS NOT DISTINCT FROM label.source_{field}" for field in fields) - sql = sql.replace(f"FROM {profit.SETTLEMENTS} s JOIN streams", - f"FROM {profit.SETTLEMENTS} s JOIN labels label ON {join} JOIN streams") - sql = sql.replace("FROM amzn_orders GROUP BY amazon_order_id", - f"FROM amzn_orders WHERE amazon_order_id IN (SELECT trim(order_id) " - f"FROM {profit.SETTLEMENTS}) GROUP BY amazon_order_id") - return sql.replace("WITH orders AS (", "WITH " + labels + "orders AS (", 1) - - -class _ValidationSources: - """Share settlement source preparation across this request's independent window sums.""" - - def __init__(self, con, bounds): - self.con, self.results, self.spans = con, {}, {} - self.bounds = [span for bound in bounds for span in (bound, *_comparisons(bound).values())] - - def _spans(self, currency=None): - if currency in self.spans: - return self.spans[currency] - spans = {(bound["start"], bound["end"]) for bound in self.bounds} - if currency is not None: - for start, end in tuple(spans): - covered = profit.settlement_coverage(self, start, end, currency) - spans.add((covered["start"] or start, covered["end"] or end)) - self.spans[currency] = sorted(spans) - return self.spans[currency] - - def _components(self, sql, params): - marker = "SELECT component, COALESCE(SUM(currency_amount), 0)" - prefix, select = sql.split(marker, 1) - aggregates, select = (marker + select).split("FROM dated WHERE", 1) - aggregates = aggregates.removeprefix("SELECT component, ") - # The grouping keys retain every input to the original window predicate and its bad-row counts. - prefix += (" , prepared(component, basis, posted, dates, bad_dates, amount, source_rows, " - "bad_amounts, unmatched_orders) AS (SELECT component, basis, posted, dates, bad_dates, " - + aggregates + " FROM dated GROUP BY component, basis, posted, dates, bad_dates) ") - spans = self._spans(params[0]) - for bound in ("lo", "hi", "lo", "hi"): - select = select.replace("CAST(? AS DATE)", "span." + bound, 1) - ranges = ", ".join(f"({index}, CAST(? AS DATE), CAST(? AS DATE))" for index in range(len(spans))) - select = f"FROM prepared JOIN (VALUES {ranges}) AS span(id, lo, hi) ON" + select - select = select.replace("GROUP BY component ORDER BY component", - "GROUP BY span.id, component ORDER BY span.id, component") - bindings = list(params[:2]) + [value for span in spans for value in span] - rows = self.con.execute(_settlement_source(prefix) + "SELECT span.id, component, " - "COALESCE(SUM(amount), 0), SUM(source_rows), SUM(bad_amounts), SUM(unmatched_orders) " - + select, bindings).fetchall() - for index, span in enumerate(spans): - self.results[(sql, tuple(params[:2]) + span * 2)] = [row[1:] for row in rows if row[0] == index] - - def _unscoped(self, sql): - marker = "AND (CAST(" - source, _date_filter = sql.split(marker, 1) - source = source.replace("SELECT count(*)", f"SELECT CAST({profit._moment_sql('posted_date')} AS DATE) AS posted", 1) - spans = self._spans(profit.economics_currency(self.con)) - sums = ["count(*) FILTER (WHERE posted BETWEEN CAST(? AS DATE) AND CAST(? AS DATE) OR posted IS NULL)" - for _span in spans] - counts = self.con.execute("SELECT " + ", ".join(sums) + " FROM (" + source + ")", - [value for span in spans for value in span]).fetchone() - for index, count in enumerate(counts): - self.results[(sql, spans[index])] = [(count,)] - - def execute(self, sql, params=()): - params = tuple(params or ()) - key = sql, params - if key in self.results: - return _SourceRows(self.results[key]) - if sql == "SELECT column_name FROM information_schema.columns WHERE table_name = ?": - self.results[key] = self.con.execute(sql, params).fetchall() - return _SourceRows(self.results[key]) - components = (sql.startswith("WITH orders AS (") and len(params) == 6 - and sql.endswith(" GROUP BY component ORDER BY component")) - unscoped = (sql.startswith(f"SELECT count(*) FROM {profit.SETTLEMENTS} s\n") - and "WHERE h.settlement_id IS NULL" in sql and len(params) == 2) - if components: - self._components(sql, params) - elif unscoped: - self._unscoped(sql) - if key in self.results: - return _SourceRows(self.results[key]) - return self.con.execute(sql, params) - - -def _account_bounds(chosen, today): - bounds = [chosen, _bounds(today.replace(day=1), today - dt.timedelta(days=1), "mtd")] - current = today.replace(day=1) - for _ in range(2): - last = current - dt.timedelta(days=1) - current = last.replace(day=1) - bounds.append(_bounds(current, last, current.strftime("%Y-%m"))) - return bounds - - def account_view(con, period=None, *, stored=None, settings=None, today=None): """Return selected, comparison and monthly account windows without refreshing stored inputs.""" today = today or profit._today() @@ -692,8 +574,6 @@ def account_view(con, period=None, *, stored=None, settings=None, today=None): "TRY_CAST(left(CAST(start_date AS VARCHAR),10) AS DATE) IS NULL").fetchone()[0]: raise profit.SourceUnavailable("Economics dates unavailable. Resync Amazon economics.") stored, settings = stored or {}, settings or {} - periods = _account_bounds(chosen, today) - con = _ValidationSources(con, periods) cache = {} def window(bounds): key = bounds["start"], bounds["end"] @@ -702,8 +582,13 @@ def account_view(con, period=None, *, stored=None, settings=None, today=None): return {**cache[key], "period": bounds} def compared(bounds): return {**window(bounds), "comparisons": {key: window(span) for key, span in _comparisons(bounds).items()}} - mtd = compared(periods[1]) - months = [compared(bounds) for bounds in periods[2:]] + current = today.replace(day=1) + mtd = compared(_bounds(current, today - dt.timedelta(days=1), "mtd")) + months = [] + for _ in range(2): + last = current - dt.timedelta(days=1) + current = last.replace(day=1) + months.append(compared(_bounds(current, last, current.strftime("%Y-%m")))) pace = {} elapsed = mtd["period"]["days"] month_days = calendar.monthrange(today.year, today.month)[1] diff --git a/platform/modules/competitor_watch.py b/platform/modules/competitor_watch.py index ac3a76e4eaed32434799b49073048bb504e1868f..92712d6fc7a053bac09cf4678c754d26a06c387e 100644 --- a/platform/modules/competitor_watch.py +++ b/platform/modules/competitor_watch.py @@ -300,7 +300,7 @@ class _TermIndex(_DocumentIndex): self.sql("SELECT (value ->> 'activeTotal')::bigint FROM {tbl} WHERE key = %s")) total_params = ([self.slug, *([display_term(search)] if search else [])] if search or tracked else [self.key(KEY_TERM_INDEX)]) - tail = ("), " + statement = self.term_sql("WITH totals(total) AS (" + total + "), " "bounds AS (SELECT total, CASE WHEN %s >= total THEN " "greatest(0, floor((total - 1)::numeric / %s)::bigint * %s) ELSE %s END AS skip FROM totals) " "SELECT total, skip, EXISTS(SELECT 1 FROM {terms} WHERE tenant = %s AND term = %s AND " + membership + "), " @@ -308,12 +308,6 @@ class _TermIndex(_DocumentIndex): + ("AND strpos(lower(label), %s) > 0 " if search else "") + 'ORDER BY ' + order + ' LIMIT %s OFFSET (SELECT skip FROM bounds)) ' "FROM bounds") - if isinstance(total, str): - statement = self.term_sql("WITH totals(total) AS (" + total + tail) - else: - # PostgreSQL's `self.sql` returns a psycopg Composed: compose it, never concatenate it as text. - from psycopg import sql as psql - statement = psql.SQL("WITH totals(total) AS (") + total + psql.SQL(self.term_sql(tail)) params = [*total_params, offset, limit, limit, offset, self.slug, selected, self.slug, *([display_term(search)] if search else []), limit] with self.connection() as con: @@ -469,25 +463,15 @@ def _ba_columns(cur): def _sqp(cur, term=None): - """Monthly SQP rows as (term, asin, impressions, end_date) for our ASINs.""" + """SQP rows as (term, asin, impressions, end_date). SQP reports our ASINs only.""" need = {"search_query_data_search_query", "asin"} cols = _columns(cur, SQP_TABLE) if cur is not None else set() if not need <= cols: return [] - if "_report_period" in cols: - period = '"_report_period" = \'MONTH\'' - elif {"start_date", "end_date"} <= cols: - start = 'TRY_CAST("start_date" AS DATE)' - end_date = 'TRY_CAST("end_date" AS DATE)' - period = f'day({start}) = 1 AND {end_date} = last_day({start})' - else: - return [] imp = ('TRY_CAST("impression_data_asin_impression_count" AS DOUBLE)' if "impression_data_asin_impression_count" in cols else "0") end = 'CAST("end_date" AS VARCHAR)' if "end_date" in cols else "''" - where = f" WHERE {period}" - if term is not None: - where += f' AND {_term_sql("search_query_data_search_query")} = ?' + where = f' WHERE {_term_sql("search_query_data_search_query")} = ?' if term is not None else "" rows = cur.execute( f'SELECT "search_query_data_search_query", "asin", {imp}, {end} FROM {SQP_TABLE}{where}', [term] if term is not None else []).fetchall() diff --git a/requirements.txt b/requirements.txt index bb0118a83c34c480c0e9c495e3be5262671d4081..45716e5201551b4cc934ae5d6e0d24f861185128 100644 --- a/requirements.txt +++ b/requirements.txt @@ -25,7 +25,6 @@ duckdb==1.5.4 # duckdb's pandas bridge imports pytz for timezone values; pandas 3 no longer installs it. pytz>=2024.1 pyyaml>=6.0 -jsonschema==4.26.0 pillow>=10.0 beautifulsoup4>=4.12 lxml>=5.0 diff --git a/web/package-lock.json b/web/package-lock.json index 29b264c9661800ff3cda10dcc3ced14ed65a8e49..5ce70eacdce18de5fa9d988738a65780c7c06adf 100644 --- a/web/package-lock.json +++ b/web/package-lock.json @@ -25,7 +25,6 @@ "@types/react": "^18.3.12", "@types/react-dom": "^18.3.1", "@vitejs/plugin-react": "^6.0.3", - "esbuild": "^0.27.0", "openapi-typescript": "^7.13.0", "oxlint": "^1.71.0", "typescript": "~6.0.2", @@ -390,448 +389,6 @@ "integrity": "sha512-30FAj7/EoJ5mwVPOWhAyCX+FPfMDrVecJAM+Iw9NRoSl4BBAQeqj4cApHHUXOVvIPgLVDsCFoz/hGD+5QQD1GQ==", "license": "MIT" }, - "node_modules/@esbuild/aix-ppc64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.27.7.tgz", - "integrity": "sha512-EKX3Qwmhz1eMdEJokhALr0YiD0lhQNwDqkPYyPhiSwKrh7/4KRjQc04sZ8db+5DVVnZ1LmbNDI1uAMPEUBnQPg==", - "cpu": [ - "ppc64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "aix" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/android-arm": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.27.7.tgz", - "integrity": "sha512-jbPXvB4Yj2yBV7HUfE2KHe4GJX51QplCN1pGbYjvsyCZbQmies29EoJbkEc+vYuU5o45AfQn37vZlyXy4YJ8RQ==", - "cpu": [ - "arm" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/android-arm64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.27.7.tgz", - "integrity": "sha512-62dPZHpIXzvChfvfLJow3q5dDtiNMkwiRzPylSCfriLvZeq0a1bWChrGx/BbUbPwOrsWKMn8idSllklzBy+dgQ==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/android-x64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.27.7.tgz", - "integrity": "sha512-x5VpMODneVDb70PYV2VQOmIUUiBtY3D3mPBG8NxVk5CogneYhkR7MmM3yR/uMdITLrC1ml/NV1rj4bMJuy9MCg==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/darwin-arm64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.27.7.tgz", - "integrity": "sha512-5lckdqeuBPlKUwvoCXIgI2D9/ABmPq3Rdp7IfL70393YgaASt7tbju3Ac+ePVi3KDH6N2RqePfHnXkaDtY9fkw==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/darwin-x64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.27.7.tgz", - "integrity": "sha512-rYnXrKcXuT7Z+WL5K980jVFdvVKhCHhUwid+dDYQpH+qu+TefcomiMAJpIiC2EM3Rjtq0sO3StMV/+3w3MyyqQ==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/freebsd-arm64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.27.7.tgz", - "integrity": "sha512-B48PqeCsEgOtzME2GbNM2roU29AMTuOIN91dsMO30t+Ydis3z/3Ngoj5hhnsOSSwNzS+6JppqWsuhTp6E82l2w==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/freebsd-x64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.27.7.tgz", - "integrity": "sha512-jOBDK5XEjA4m5IJK3bpAQF9/Lelu/Z9ZcdhTRLf4cajlB+8VEhFFRjWgfy3M1O4rO2GQ/b2dLwCUGpiF/eATNQ==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-arm": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.27.7.tgz", - "integrity": "sha512-RkT/YXYBTSULo3+af8Ib0ykH8u2MBh57o7q/DAs3lTJlyVQkgQvlrPTnjIzzRPQyavxtPtfg0EopvDyIt0j1rA==", - "cpu": [ - "arm" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-arm64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.27.7.tgz", - "integrity": "sha512-RZPHBoxXuNnPQO9rvjh5jdkRmVizktkT7TCDkDmQ0W2SwHInKCAV95GRuvdSvA7w4VMwfCjUiPwDi0ZO6Nfe9A==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-ia32": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.27.7.tgz", - "integrity": "sha512-GA48aKNkyQDbd3KtkplYWT102C5sn/EZTY4XROkxONgruHPU72l+gW+FfF8tf2cFjeHaRbWpOYa/uRBz/Xq1Pg==", - "cpu": [ - "ia32" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-loong64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.27.7.tgz", - "integrity": "sha512-a4POruNM2oWsD4WKvBSEKGIiWQF8fZOAsycHOt6JBpZ+JN2n2JH9WAv56SOyu9X5IqAjqSIPTaJkqN8F7XOQ5Q==", - "cpu": [ - "loong64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-mips64el": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.27.7.tgz", - "integrity": "sha512-KabT5I6StirGfIz0FMgl1I+R1H73Gp0ofL9A3nG3i/cYFJzKHhouBV5VWK1CSgKvVaG4q1RNpCTR2LuTVB3fIw==", - "cpu": [ - "mips64el" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-ppc64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.27.7.tgz", - "integrity": "sha512-gRsL4x6wsGHGRqhtI+ifpN/vpOFTQtnbsupUF5R5YTAg+y/lKelYR1hXbnBdzDjGbMYjVJLJTd2OFmMewAgwlQ==", - "cpu": [ - "ppc64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-riscv64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.27.7.tgz", - "integrity": "sha512-hL25LbxO1QOngGzu2U5xeXtxXcW+/GvMN3ejANqXkxZ/opySAZMrc+9LY/WyjAan41unrR3YrmtTsUpwT66InQ==", - "cpu": [ - "riscv64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-s390x": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.27.7.tgz", - "integrity": "sha512-2k8go8Ycu1Kb46vEelhu1vqEP+UeRVj2zY1pSuPdgvbd5ykAw82Lrro28vXUrRmzEsUV0NzCf54yARIK8r0fdw==", - "cpu": [ - "s390x" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-x64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.27.7.tgz", - "integrity": "sha512-hzznmADPt+OmsYzw1EE33ccA+HPdIqiCRq7cQeL1Jlq2gb1+OyWBkMCrYGBJ+sxVzve2ZJEVeePbLM2iEIZSxA==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/netbsd-arm64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.27.7.tgz", - "integrity": "sha512-b6pqtrQdigZBwZxAn1UpazEisvwaIDvdbMbmrly7cDTMFnw/+3lVxxCTGOrkPVnsYIosJJXAsILG9XcQS+Yu6w==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "netbsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/netbsd-x64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.27.7.tgz", - "integrity": "sha512-OfatkLojr6U+WN5EDYuoQhtM+1xco+/6FSzJJnuWiUw5eVcicbyK3dq5EeV/QHT1uy6GoDhGbFpprUiHUYggrw==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "netbsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/openbsd-arm64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.27.7.tgz", - "integrity": "sha512-AFuojMQTxAz75Fo8idVcqoQWEHIXFRbOc1TrVcFSgCZtQfSdc1RXgB3tjOn/krRHENUB4j00bfGjyl2mJrU37A==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "openbsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/openbsd-x64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.27.7.tgz", - "integrity": "sha512-+A1NJmfM8WNDv5CLVQYJ5PshuRm/4cI6WMZRg1by1GwPIQPCTs1GLEUHwiiQGT5zDdyLiRM/l1G0Pv54gvtKIg==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "openbsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/openharmony-arm64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.27.7.tgz", - "integrity": "sha512-+KrvYb/C8zA9CU/g0sR6w2RBw7IGc5J2BPnc3dYc5VJxHCSF1yNMxTV5LQ7GuKteQXZtspjFbiuW5/dOj7H4Yw==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "openharmony" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/sunos-x64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.27.7.tgz", - "integrity": "sha512-ikktIhFBzQNt/QDyOL580ti9+5mL/YZeUPKU2ivGtGjdTYoqz6jObj6nOMfhASpS4GU4Q/Clh1QtxWAvcYKamA==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "sunos" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/win32-arm64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.27.7.tgz", - "integrity": "sha512-7yRhbHvPqSpRUV7Q20VuDwbjW5kIMwTHpptuUzV+AA46kiPze5Z7qgt6CLCK3pWFrHeNfDd1VKgyP4O+ng17CA==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/win32-ia32": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.27.7.tgz", - "integrity": "sha512-SmwKXe6VHIyZYbBLJrhOoCJRB/Z1tckzmgTLfFYOfpMAx63BJEaL9ExI8x7v0oAO3Zh6D/Oi1gVxEYr5oUCFhw==", - "cpu": [ - "ia32" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/win32-x64": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.27.7.tgz", - "integrity": "sha512-56hiAJPhwQ1R4i+21FVF7V8kSD5zZTdHcVuRFMW0hn753vVfQN8xlx4uOPT4xoGH0Z/oVATuR82AiqSTDIpaHg==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=18" - } - }, "node_modules/@glideapps/glide-data-grid": { "version": "6.0.3", "resolved": "https://registry.npmjs.org/@glideapps/glide-data-grid/-/glide-data-grid-6.0.3.tgz", @@ -2292,48 +1849,6 @@ "integrity": "sha512-toUI84YS5YmxW219erniWD0CIVOo46xGKColeNQRgOzDorgBi1v4D71/OFzgD9GO2UGKIv1C3Sp8DAn0+j5w7A==", "license": "MIT" }, - "node_modules/esbuild": { - "version": "0.27.7", - "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.27.7.tgz", - "integrity": "sha512-IxpibTjyVnmrIQo5aqNpCgoACA/dTKLTlhMHihVHhdkxKyPO1uBBthumT0rdHmcsk9uMonIWS0m4FljWzILh3w==", - "dev": true, - "hasInstallScript": true, - "license": "MIT", - "bin": { - "esbuild": "bin/esbuild" - }, - "engines": { - "node": ">=18" - }, - "optionalDependencies": { - "@esbuild/aix-ppc64": "0.27.7", - "@esbuild/android-arm": "0.27.7", - "@esbuild/android-arm64": "0.27.7", - "@esbuild/android-x64": "0.27.7", - "@esbuild/darwin-arm64": "0.27.7", - "@esbuild/darwin-x64": "0.27.7", - "@esbuild/freebsd-arm64": "0.27.7", - "@esbuild/freebsd-x64": "0.27.7", - "@esbuild/linux-arm": "0.27.7", - "@esbuild/linux-arm64": "0.27.7", - "@esbuild/linux-ia32": "0.27.7", - "@esbuild/linux-loong64": "0.27.7", - "@esbuild/linux-mips64el": "0.27.7", - "@esbuild/linux-ppc64": "0.27.7", - "@esbuild/linux-riscv64": "0.27.7", - "@esbuild/linux-s390x": "0.27.7", - "@esbuild/linux-x64": "0.27.7", - "@esbuild/netbsd-arm64": "0.27.7", - "@esbuild/netbsd-x64": "0.27.7", - "@esbuild/openbsd-arm64": "0.27.7", - "@esbuild/openbsd-x64": "0.27.7", - "@esbuild/openharmony-arm64": "0.27.7", - "@esbuild/sunos-x64": "0.27.7", - "@esbuild/win32-arm64": "0.27.7", - "@esbuild/win32-ia32": "0.27.7", - "@esbuild/win32-x64": "0.27.7" - } - }, "node_modules/escalade": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", diff --git a/web/package-lock.linux.sha256 b/web/package-lock.linux.sha256 index e63f7a0b4481302fb298fbe769e767faac24f540..daa34af70b65b397fc83a12fdcec3ae911ff70a3 100644 --- a/web/package-lock.linux.sha256 +++ b/web/package-lock.linux.sha256 @@ -1,2 +1,2 @@ -9cd826154f5fecd9784f6632ad62e9da94db63b1ed343ab137fbd29ce5690c07 package-lock.json -# resolved on linux/x64 by HF job 6afd11b54643bac7ac631093 on 2026-11-18, npm ci passed there before upload. Regenerate ONLY with lockfile_linux.py (W-170). +31cb494538043db42bdf698846c2262f5eb234fb686aa03d9cf3ab1cbfdbe2b8 package-lock.json +# resolved on linux/x64 in the SDLC 72 lane A HF container on 2026-10-09 (S72-T05, npm install --save-exact @phosphor-icons/react@2.1.10); npm ci --ignore-scripts passed there (228 packages). Regenerate ONLY with lockfile_linux.py (W-170). diff --git a/web/package.json b/web/package.json index a8eadcac73854da83c0cdf4d02f6865087b0889d..899f0c9da672b6ff210e4762a6a020881243a048 100644 --- a/web/package.json +++ b/web/package.json @@ -39,7 +39,6 @@ "@types/react": "^18.3.12", "@types/react-dom": "^18.3.1", "@vitejs/plugin-react": "^6.0.3", - "esbuild": "^0.27.0", "openapi-typescript": "^7.13.0", "oxlint": "^1.71.0", "typescript": "~6.0.2", diff --git a/web/public/AIOS_DEPLOY_RECEIPT b/web/public/AIOS_DEPLOY_RECEIPT index d02d5bf3cab15a9341435ae02d6872642077c394..ddb1e2ba949b647ec26c5678ba11c9ad88c54edc 100644 --- a/web/public/AIOS_DEPLOY_RECEIPT +++ b/web/public/AIOS_DEPLOY_RECEIPT @@ -1 +1 @@ -7c3abd53bb8053c1ff1dee68532366490bc632e10176ced049a6e6c163b5f300 +f4813286cc57af9ab94a458dafeabfd9f77075e08fa73b573fd34b3ec376779d diff --git a/web/src/apps/amazon-profit/AppView.tsx b/web/src/apps/amazon-profit/AppView.tsx index d3d8af591b0667c26e358f81ee86f602a34c1ec7..a3d252bd168235097df3c1d09f37ed6ca4efd040 100644 --- a/web/src/apps/amazon-profit/AppView.tsx +++ b/web/src/apps/amazon-profit/AppView.tsx @@ -2,7 +2,7 @@ import { useCallback, useEffect, useMemo, useState } from "react"; import { API_V1, CREDENTIALS, ROWS_STALE_EVENT, signal } from "../../apiContract"; import { ACCESS_REFUSED_MESSAGE, AppTableGrid, clearTopicRowsCache, dropTableRowsCache, OverlayProvider, LP_BLUE_DEEP, LP_GREEN_DEEP } from "../../customer-grid/seam"; -import type { AppCellDraw, AppTableHost, AppViewProps, Row } from "../../customer-grid/seam"; +import type { AppTableHost, AppViewProps } from "../../customer-grid/seam"; import { AMAZON_DATA_ENTRY_TABLE, AMAZON_PROFIT_TABLE, AmazonToolbar, COMMAND_CENTER_TABLES, CommandCenterTabStrip, CommandCenterTableContext } from "./AmazonPeriodPicker"; import { LogoLoader } from "../../ui/LogoLoader"; @@ -11,36 +11,12 @@ import PicturePopup from "./PicturePopup"; import { ProjectsPopup, ProjectsPage, projectsTargetFromRow } from "./Projects"; import type { ProjectsTarget } from "./Projects"; import { CompetitorsPopup, competitorsTargetFromRow } from "./Competitors"; -import GridFilter, { SIGNAL_SORTS } from "./pdp/GridFilter"; -import HistoryPopup from "./pdp/HistoryPopup"; -import PdpUpdatePopup, { type PdpListing, type PdpNavigation } from "./pdp/PdpUpdatePopup"; -import WorkflowWindow from "./pdp/WorkflowWindow"; -import type { PdpProjectNavigation } from "./pdp/ContinueWorkflow"; -import type { PdpProjectPane } from "./pdp/contract"; import "../../styles/sdlc81/command-center-app.css"; -import "../../styles/sdlc86/project-manager-pdp.css"; const PROJECT_PROGRESS_COLORS = { inProgress: LP_BLUE_DEEP, completed: LP_GREEN_DEEP }; const PROJECT_PROGRESS_TABLES = new Set([AMAZON_PROFIT_TABLE, "ut_amazon_asin_profit", "ut_amazon_parent_profit"]); /** The App's 56 px rows (R5). Table keeps its own default. */ const COMMAND_CENTER_COLUMNS = { rowHeight: 56 }; -const ASIN_COLUMNS = { rowHeight: 56, headerTone: { projects: "primary", pdp_update: "primary", history: "primary" }, - widths: { asin: 150, projects: 210, pdp_update: 170, trend_score: 150, search_trend: 210, - market_share_trend: 240, click_share_trend: 240, exposure: 190, funnel_score: 150, history: 150 } } as const; -const noReport = (reasonKey: string) => ({ reasonKey, equals: "No search report", text: "No search report" }); -const signalTrend = (key: string, share: boolean): AppCellDraw => ({ kind: "trend", seriesKey: key + "_series", - valueKey: key + "_now", changeKey: key, format: share ? "percent" : "number", showValue: share, - omitMissingValue: true, valuePrecision: "adaptive", changeDecimals: 0, changeNeutralBand: .5, fitLabels: true, barTone: "primary", - unavailable: noReport(key + "_reason") }); -const SIGNAL_CELLS: AppTableHost["cellRenderers"] = { - trend_score: { draw: { kind: "ring", labelPosition: "after", unavailable: noReport("trend_score_reason") } }, - search_trend: { draw: signalTrend("search_trend", false) }, - market_share_trend: { draw: signalTrend("market_share_trend", true) }, - click_share_trend: { draw: signalTrend("click_share_trend", true) }, - exposure: { draw: { kind: "badge", subKey: "exposure_label", emptyText: "None", unavailable: noReport("exposure_reason") } }, - funnel_score: { draw: { kind: "score", neutralRange: [4.5, 5.5], outerRange: [3, 7], emptyText: "Few", - unavailable: noReport("funnel_score_reason") } }, -}; /** Admission belongs to the selected tab's fresh probe. */ type TableProbe = { table: string; status: "ready"; tables: Record } @@ -48,17 +24,12 @@ type TableProbe = { table: string; status: "ready"; tables: Record(null); const [picture, setPicture] = useState<{ ref: string; label: string } | null>(null); - const [projects, setProjects] = useState<(ProjectsTarget & { - selectedProjectId?: string; originListing?: PdpListing; initialPane?: PdpProjectPane; - }) | null>(null); + const [projects, setProjects] = useState(null); const [competitors, setCompetitors] = useState(null); - const [history, setHistory] = useState(null); - const [pdp, setPdp] = useState(null); - const [workflow, setWorkflow] = useState<{ id: string; listing: PdpListing; resumeRevision?: number } | null>(null); const [gridRevision, setGridRevision] = useState(0); const [entryEmpty, setEntryEmpty] = useState<{ table: string; empty: boolean } | null>(null); const openTable = useCallback((next: string) => { @@ -69,9 +40,6 @@ export default function ProjectManagementView(props: AppViewProps & PdpNavigatio setPicture(null); setProjects(null); setCompetitors(null); - setHistory(null); - setPdp(null); - setWorkflow(null); setTable(next); }, [table]); const reloadGrid = useCallback(() => { @@ -120,37 +88,12 @@ export default function ProjectManagementView(props: AppViewProps & PdpNavigatio }, [dataEntry, admitted, table]); const entryChecked = !dataEntry || notBuilt || entryEmpty?.table === table; const allComplete = dataEntry && admitted && entryEmpty?.table === table && entryEmpty.empty; - const openProject = (id: string, asin: string, listing: PdpListing, initialPane?: PdpProjectPane) => { - if (props.onOpenProject) { props.onOpenProject(id, asin); return; } - setPdp(null); - setProjects({ grain: "asin", rowKey: asin, title: listing.title, selectedProjectId: id, originListing: listing, initialPane }); - }; - const resumeWorkflow = () => { - setProjects(null); - setWorkflow(current => current ? { ...current, resumeRevision: (current.resumeRevision ?? 0) + 1 } : current); - }; - const projectNavigation: PdpProjectNavigation = { onOpenWorkflow: (next, item, target) => { - if (workflow?.id === next.id && workflow.listing.asin === item.asin) { - resumeWorkflow(); - return; - } - setProjects(null); - if (props.onOpenWorkflow) { props.onOpenWorkflow(next.id, item.asin); return; } - setWorkflow({ id: next.id, listing: { asin: item.asin, title: target?.title ?? "", sku: target?.sku ?? null, brand: null } }); - } }; // C2: progress colours, button doors and the Picture popup, declared once per table. // Only the Projects table keeps New record (`append`). const cellRenderers = useMemo(() => ({ - ...(table === "ut_amazon_asin_profit" ? SIGNAL_CELLS : {}), "*": { progressColors: PROJECT_PROGRESS_TABLES.has(table) ? PROJECT_PROGRESS_COLORS : undefined }, projects: { button: true, activate: row => setProjects(projectsTargetFromRow(table, row)) }, competitors: { button: true, activate: row => setCompetitors(competitorsTargetFromRow(table, row)) }, - history: { button: true, activate: row => setHistory(row) }, - pdp_update: { button: true, activate: row => { - if (!row.asin) return false; - setPdp({ asin: String(row.asin), title: String(row.title || ""), - sku: row.sku ? String(row.sku) : null, brand: row.brand ? String(row.brand) : null }); - } }, "type:image": { activate: (record, field) => { const ref = String(record[field.key] ?? "").trim(); if (!ref) return false; @@ -172,35 +115,17 @@ export default function ProjectManagementView(props: AppViewProps & PdpNavigatio : table === "ut_amazon_projects" ?
- +
: !entryChecked ? : : undefined} - sortChoices={table === "ut_amazon_asin_profit" ? SIGNAL_SORTS : undefined} + columns={COMMAND_CENTER_COLUMNS} cellRenderers={cellRenderers} toolbar={} />} {picture && setPicture(null)} />} - {projects && setProjects(null)} onChanged={reloadGrid} - backLabel={projects.initialPane ? "Back to workflow" : projects.originListing ? "Back to projects" : undefined} - onBack={projects.initialPane ? resumeWorkflow : projects.originListing - ? () => { setPdp(projects.originListing ?? null); setProjects(null); setWorkflow(null); } : undefined} />} + {projects && setProjects(null)} onChanged={reloadGrid} />} {competitors && setCompetitors(null)} onChanged={reloadGrid} />} - {history && setHistory(null)} onChanged={reloadGrid} />} - {pdp && setPdp(null)} onChanged={reloadGrid} - onOpenWorkflow={props.onOpenWorkflow ?? ((id, asin) => { - setWorkflow({ id, listing: { ...pdp, asin } }); setPdp(null); - })} onAddWorkflow={props.onAddWorkflow} - onOpenProject={(id, asin) => openProject(id, asin, pdp)} />} - {workflow && setWorkflow(null)} onChanged={reloadGrid} - onBack={() => { setPdp(workflow.listing); setWorkflow(null); }} - onOpenProject={() => openProject(workflow.id, workflow.listing.asin, workflow.listing)} - onOpenProjectPane={pane => openProject(workflow.id, workflow.listing.asin, workflow.listing, pane)} />} ; } diff --git a/web/src/apps/amazon-profit/ProjectOverview.tsx b/web/src/apps/amazon-profit/ProjectOverview.tsx index 7bbf8cf74fa4e814215c9c259d84ce773a6ac55a..46d7a996abb4aa50c68be723a073429ad3e87265 100644 --- a/web/src/apps/amazon-profit/ProjectOverview.tsx +++ b/web/src/apps/amazon-profit/ProjectOverview.tsx @@ -11,8 +11,6 @@ import { X } from "@phosphor-icons/react/dist/csr/X"; import { useEscapeClose } from "../../ui/useEscapeClose"; import ProjectTasks from "./ProjectTasks"; import ProjectSkuPicker from "./ProjectSkuPicker"; -import { PdpApprovalRow, PdpOverviewRows, PdpStep, type ProjectPdp } from "./pdp/ContinueWorkflow"; -import { removeItems } from "./pdp/api"; import { EMPTY_FILTER, GRAINS, ProductToolbar, SelectBox, TABLES, normalizeProducts, number, readPage, scopedNodes, type Field, type FilterTree, type Row, type Sorts, @@ -238,12 +236,11 @@ function ProjectDeleteDialog({ name, busy, error, onCancel, onConfirm }: { } export default function ProjectOverview({ id, detail, canEdit: mayEdit, onChanged, onDeleted, onCancel, - error, setError, setDetail, setTick, formId, active, rowScope = null, onClearRowScope, pdp }: { + error, setError, setDetail, setTick, formId, active, rowScope = null, onClearRowScope }: { id: string; detail: Detail; canEdit: boolean; onChanged: () => void; onDeleted: () => void; onCancel: () => void; error: string; setError: Dispatch>; setDetail: Dispatch>; setTick: Dispatch>; formId: string; active: boolean; rowScope?: RowScope | null; onClearRowScope?: () => void; - pdp?: ProjectPdp; }) { const [draft, setDraft] = useState(() => overviewDraftOf(detail.project)); const [people, setPeople] = useState([]); @@ -321,12 +318,6 @@ export default function ProjectOverview({ id, detail, canEdit: mayEdit, onChange setBusy(true); const fallback = "Products were not removed."; try { - if (pdp) { - if (!pdp.detail?.canEdit) throw new Error("The workflow is unavailable. Reload the project."); - pdp.accept((await removeItems(id, pairs.map(pair => pair.key))).workflow); - setTick(value => value + 1); - return { refused: [] }; - } const answer = await call(`/${enc(id)}/targets/remove`, { method: "POST", body: { targets: pairs } }, fallback); if (!answer?.project || !Array.isArray(answer.targets)) throw new Error("The remove answer is incomplete. Reload the project."); @@ -352,10 +343,10 @@ export default function ProjectOverview({ id, detail, canEdit: mayEdit, onChange writing.current = false; setBusy(false); } - }, [detail?.canEdit, id, mayEdit, onChanged, setDetail, setError, setTick, pdp]); + }, [detail?.canEdit, id, mayEdit, onChanged, setDetail, setError, setTick]); const { project, targets } = detail; - const canEdit = mayEdit && detail.canEdit && (!pdp || Boolean(pdp.detail?.canEdit)); + const canEdit = mayEdit && detail.canEdit; const deleteOpen = confirming && canEdit && active; useEffect(() => { if (!canEdit || !active) { setConfirming(false); setAdding(false); } @@ -469,7 +460,6 @@ export default function ProjectOverview({ id, detail, canEdit: mayEdit, onChange
- {project.type === "pdp" && pdp ? : null} {canEdit ? set("status", value)} /> @@ -492,7 +482,6 @@ export default function ProjectOverview({ id, detail, canEdit: mayEdit, onChange onChange={values => { set("ownerUsername", values[0] ?? null); set("owner", null); }} /> : project.owner || project.ownerUsername ? : No owner} - {project.type === "pdp" && pdp ? : null}
{contributors.map(person => @@ -566,7 +555,7 @@ export default function ProjectOverview({ id, detail, canEdit: mayEdit, onChange : null}
{canEdit && active && adding - ? setAdding(false)} pdp={pdp} + ? setAdding(false)} onAdded={answer => { setDetail(current => current ? { ...current, project: answer.project, targets: answer.targets, cause: answer.cause } : current); @@ -577,7 +566,7 @@ export default function ProjectOverview({ id, detail, canEdit: mayEdit, onChange : null} {targets.length ? + onRemove={removeTargets} /> : null}
@@ -665,10 +654,9 @@ async function readLinked(table: string, grain: ProjectGrain, tree: FilterTree, } } -function ProjectSkuList({ targets: linked, scope, canEdit, busy, active, onRemove, pdp }: { +function ProjectSkuList({ targets: linked, scope, canEdit, busy, active, onRemove }: { targets: Target[]; scope: (RowScope & { clear: () => void }) | null; canEdit: boolean; busy: boolean; active: boolean; onRemove: (pairs: TargetPair[]) => Promise; - pdp?: ProjectPdp; }) { const id = useId(); const [chosen, setChosen] = useState("sku"); @@ -698,14 +686,13 @@ function ProjectSkuList({ targets: linked, scope, canEdit, busy, active, onRemov
: null} target.grain === grain)} canEdit={canEdit} busy={busy} active={active} - onRemove={onRemove} pdp={pdp} /> + onRemove={onRemove} /> ; } -function SkuGrainList({ grain, panelId, tabId, targets, canEdit, busy, active, onRemove, pdp }: { +function SkuGrainList({ grain, panelId, tabId, targets, canEdit, busy, active, onRemove }: { grain: ProjectGrain; panelId: string; tabId: string | null; targets: Target[]; canEdit: boolean; busy: boolean; active: boolean; onRemove: (pairs: TargetPair[]) => Promise; - pdp?: ProjectPdp; }) { const table = TABLES[grain]; const label = GRAIN_LABEL[grain]; @@ -799,7 +786,7 @@ function SkuGrainList({ grain, panelId, tabId, targets, canEdit, busy, active, o : new Set(Array.from(current).filter(key => shown.has(key)))); }, [term, blocked, shownSignature]); const columns = LIST_COLUMNS[grain]; - const span = (canEdit ? 1 : 0) + 1 + columns.length + (pdp ? 2 : 0); + const span = (canEdit ? 1 : 0) + 1 + columns.length; const resetView = () => { setSelected(new Set()); setRefused([]); setRemoveError(""); }; const clearView = () => { resetView(); setFilters(EMPTY_FILTER); setSorts([]); setGroup(null); setSearch(""); }; const viewSet = filtered || sorts.length > 0 || group !== null || selected.size > 0 || search !== ""; @@ -864,7 +851,6 @@ function SkuGrainList({ grain, panelId, tabId, targets, canEdit, busy, active, o {canEdit ? {boxFor(visibleKeys, `Select all shown ${plural}`)} : null} Product {columns.map(column => {column.label})} - {pdp ? <>StepWorkflow : null} {groups.map(([value, members]) => {group !== null ? @@ -887,7 +873,6 @@ function SkuGrainList({ grain, panelId, tabId, targets, canEdit, busy, active, o return {shown || Not set}; })} - {pdp ? : null} ; })} )} @@ -913,14 +898,6 @@ function SkuGrainList({ grain, panelId, tabId, targets, canEdit, busy, active, o ; } -function PdpSkuCells({ pdp, target }: { pdp: ProjectPdp; target: Target }) { - const asin = target.grain === "asin" ? target.key : target.asin; - const item = pdp.detail?.workflow.items.find(item => item.asin === asin); - return <>{item ? : Unavailable} - ; -} - export function ProjectForm({ targets, onCreated, onCancel, onForbidden }: { targets: Array<{ grain: ProjectGrain; key: string }>; onCreated: (project: Project) => void; onCancel: () => void; onForbidden: () => void; diff --git a/web/src/apps/amazon-profit/ProjectSkuPicker.tsx b/web/src/apps/amazon-profit/ProjectSkuPicker.tsx index b8bff2a7eea1f0e4dfbc31652e7d9895a81c3999..9e2f4f7d2981e043d432174fad3a15fc14c74c3f 100644 --- a/web/src/apps/amazon-profit/ProjectSkuPicker.tsx +++ b/web/src/apps/amazon-profit/ProjectSkuPicker.tsx @@ -11,9 +11,6 @@ import { FilterBuilderPanel, Popover, SortBuilder, countConditions, import { CenteredPopup, GRAIN_LABEL, ProjectRequestError, call, enc, messageOf, pictureSrc, text, type Project, type ProjectGrain, type Target } from "./ProjectsShared"; import "../../styles/sdlc85/project-sku-picker.css"; -import { projectClaimNames, readProjectClaims, type ProjectPdp } from "./pdp/ContinueWorkflow"; -import { addItems, PdpRequestError } from "./pdp/api"; -import type { PdpClaim } from "./pdp/contract"; export type FilterTree = FilterBuilderPanelProps["filters"]; type FilterNode = FilterTree["nodes"][number]; @@ -175,7 +172,7 @@ export type TaskTargetSource = { name: string; targets?: ReadonlyArray<{ grain: string; key: string }> | null; targetFilter?: { grain: string; filter: unknown } | null; completionFilter?: unknown; }; -type PickerBase = { projectId: string; canEdit: boolean; onClose: () => void; pdp?: ProjectPdp }; +type PickerBase = { projectId: string; canEdit: boolean; onClose: () => void }; export type SkuPickerProps = PickerBase & ( | { mode: "add"; existing: Target[]; onAdded: (answer: ProjectSkuAddAnswer) => void } | { mode: "task"; task: TaskTargetSource; onAssign: (change: TaskTargetChange) => Promise }); @@ -204,12 +201,11 @@ function taskStart(task: TaskTargetSource | null) { done: treeOf(task?.completionFilter), picked, assigned: !!saved || Object.keys(picked).length > 0 }; } -export default function ProjectSkuPicker({ projectId, existing, canEdit, onAdded, onClose, pdp }: { +export default function ProjectSkuPicker({ projectId, existing, canEdit, onAdded, onClose }: { projectId: string; existing: Target[]; canEdit: boolean; onAdded: (answer: ProjectSkuAddAnswer) => void; onClose: () => void; - pdp?: ProjectPdp; }) { - return ; + return ; } /** One picker body: `add` links products to the project, `task` sets one task's targets and completion filter. */ @@ -217,15 +213,11 @@ export function SkuPicker(props: SkuPickerProps) { const { projectId, canEdit, onClose } = props; const task = props.mode === "task" ? props : null; const taskMode = task !== null; - const pdp = props.mode === "add" ? props.pdp : undefined; - const pdpMode = Boolean(pdp); - const workflow = pdp?.detail?.workflow; - const [taken, setTaken] = useState([]); const existing = props.mode === "add" ? props.existing : NO_TARGETS; const onAdded = props.mode === "add" ? props.onAdded : ignoreAnswer; const id = useId(); const [start] = useState(() => taskStart(task?.task ?? null)); - const [grain, setGrain] = useState(pdp ? "asin" : start.grain); + const [grain, setGrain] = useState(start.grain); const [query, setQuery] = useState(""); const [search, setSearch] = useState(""); const [filters, setFilters] = useState(start.filter); @@ -269,17 +261,15 @@ export function SkuPicker(props: SkuPickerProps) { const order = useMemo(() => group ? [{ colId: group, dir: "asc" }, ...sorts.filter(sort => sort.colId !== group)] : sorts, [group, sorts]); const available = Math.max(0, (match?.total ?? 0) - (match?.linked ?? 0)); - const holder = (key: string) => workflow ? projectClaimNames(taken, workflow, key) : ""; - const selectedCount = allMatching ? Math.max(0, available - excluded.size) - : workflow ? Array.from(selected).filter(key => !holder(key)).length : selected.size; - const ready = canEdit && (!pdp || Boolean(pdp.detail?.canEdit)) && !forbidden && !loading && !busy && query === search && match !== null && !error; + const selectedCount = allMatching ? Math.max(0, available - excluded.size) : selected.size; + const ready = canEdit && !forbidden && !loading && !busy && query === search && match !== null && !error; const isSelected = (key: string) => linked.has(key) || (allMatching ? !excluded.has(key) : selected.has(key)); - const pageKeys = rows.map(row => text(row[grain])).filter(key => key && !linked.has(key) && !holder(key)); + const pageKeys = rows.map(row => text(row[grain])).filter(key => key && !linked.has(key)); const checkedOnPage = pageKeys.filter(isSelected).length; const preview = taskMode && kind === "filter"; const picks = selected.size + Object.entries(held) .reduce((sum, [value, keys]) => value === grain ? sum : sum + (keys?.length ?? 0), 0); - const grains = pdp ? ["asin"] as const : taskMode ? TASK_GRAINS : GRAINS; + const grains = taskMode ? TASK_GRAINS : GRAINS; const resetSelection = () => { setSelected(new Set()); setAllMatching(false); setExcluded(new Set()); }; const resetRead = () => { @@ -335,11 +325,6 @@ export function SkuPicker(props: SkuPickerProps) { if (counted && counted.total > 0 && !products.length) { throw new Error("Matching products did not load. Retry to refresh the matches."); } - if (pdpMode) { - const claims = await readProjectClaims(products.map(row => text(row.asin)), controller.signal); - if (controller.signal.aborted) return; - setTaken(current => cursor === null ? claims : [...current, ...claims]); - } setFields(page.fields); setRows(current => { if (counted?.total === 0) return []; @@ -370,7 +355,7 @@ export function SkuPicker(props: SkuPickerProps) { }) .finally(() => { if (!controller.signal.aborted) setLoading(false); }); return () => controller.abort(); - }, [projectId, canEdit, grain, table, tree, order, cursor, attempt, linkedSignature, taskMode, pdpMode]); + }, [projectId, canEdit, grain, table, tree, order, cursor, attempt, linkedSignature, taskMode]); const toggleKeys = (keys: string[], checked: boolean) => { if (allMatching) setExcluded(current => { @@ -389,11 +374,6 @@ export function SkuPicker(props: SkuPickerProps) { if (!ready || !selectedCount || submitting.current) return; submitting.current = true; setBusy(true); setRefused([]); try { - if (pdp) { - pdp.accept((await addItems(projectId, Array.from(selected).filter(key => !holder(key)))).workflow); - onClose(); - return; - } // Exclusions apply after server ranking, so they cannot promote a different top-N row. const complete = allMatching && excluded.size ? await matchingKeys(projectId, grain, tree, match!.total) : null; @@ -413,11 +393,7 @@ export function SkuPicker(props: SkuPickerProps) { } else onClose(); } catch (reason) { setError(messageOf(reason, "Products were not added.")); - if (reason instanceof PdpRequestError && reason.status === 409) { - try { setTaken(await readProjectClaims(rows.map(row => text(row.asin)))); } - catch (refreshReason) { setError(messageOf(refreshReason, "Workflow claims did not load. Retry.")); } - } - if ((reason instanceof ProjectRequestError || reason instanceof PdpRequestError) && [401, 403].includes(reason.status)) { + if (reason instanceof ProjectRequestError && [401, 403].includes(reason.status)) { setForbidden(true); setRows([]); setFields([]); setMatch(null); } } finally { submitting.current = false; setBusy(false); } @@ -438,10 +414,7 @@ export function SkuPicker(props: SkuPickerProps) { try { const every = await matchingKeys(projectId, grain, tree, current.total); if (every.total !== current.total) throw new Error("Products changed. Retry to refresh the matches."); - const claims = workflow ? await readProjectClaims(every.keys) : []; - if (workflow) setTaken(claims); - const eligible = workflow ? every.keys.filter(key => !linked.has(key) && !projectClaimNames(claims, workflow, key)) : every.keys; - setSelected(previous => new Set([...previous, ...eligible])); + setSelected(previous => new Set([...previous, ...every.keys])); } catch (reason) { refuse(reason, "Matches did not load."); } finally { submitting.current = false; setBusy(false); } }; @@ -535,7 +508,7 @@ export function SkuPicker(props: SkuPickerProps) { {match?.linked ? ` · ${number(match.linked)} already linked` : ""} {match && available > 0 && !preview ? )} @@ -114,19 +97,15 @@ function ProjectDetail({ id, canEdit: mayEdit, rowScope: openedFrom, onChanged, ; const { project } = detail; const canEdit = mayEdit && detail.canEdit; - const scopedAsins = rowScope ? [...new Set(detail.targets.filter(target => inRowScope(target, rowScope)) - .flatMap(target => target.grain === "asin" ? [target.key] : target.asin ? [target.asin] : []))] : []; return
- {pdp ? : null}{headerExtra} {rail}
- {project.type === "pdp" - ? - : } - {extras.map(extra => )} +
; } // ---- The popup a Projects cell opens ---- -export interface ProjectsPopupProps extends ProjectExtras { +export interface ProjectsPopupProps { grain: ProjectGrain; rowKey: string; title?: string; - selectedProjectId?: string; onClose: () => void; onChanged?: () => void; } @@ -160,13 +132,11 @@ export interface ProjectsPopupProps extends ProjectExtras { type PopupView = { kind: "list" } | { kind: "detail"; id: string } | { kind: "new" } | { kind: "add" }; export function ProjectsPopup(props: ProjectsPopupProps) { - return ; + return ; } -function ProjectsPopupContent({ grain, rowKey, title, selectedProjectId, onClose, onChanged, - pdp, backLabel, onBack, headerExtra, extraPanes, initialPane }: ProjectsPopupProps) { - const [view, setView] = useState(() => selectedProjectId - ? { kind: "detail", id: selectedProjectId } : { kind: "list" }); +function ProjectsPopupContent({ grain, rowKey, title, onClose, onChanged }: ProjectsPopupProps) { + const [view, setView] = useState({ kind: "list" }); const [read, setRead] = useState(null); const [error, setError] = useState(""); const [tick, setTick] = useState(0); @@ -186,7 +156,7 @@ function ProjectsPopupContent({ grain, rowKey, title, selectedProjectId, onClose const label = `${GRAIN_LABEL[grain]} ${rowKey}`; const actions = view.kind === "detail" - ? + ? : view.kind === "list" && canEdit ? <> @@ -213,8 +183,6 @@ function ProjectsPopupContent({ grain, rowKey, title, selectedProjectId, onClose : null} {view.kind === "detail" ? setView({ kind: "list" })} /> : null} {view.kind === "new" && canEdit @@ -236,7 +204,7 @@ function ProjectsPopupContent({ grain, rowKey, title, selectedProjectId, onClose // ---- The Projects tab: every project, the same rows ---- -export function ProjectsPage({ pdp, headerExtra, extraPanes }: ProjectExtras = {}) { +export function ProjectsPage() { const [read, setRead] = useState(null); const [error, setError] = useState(""); const [tick, setTick] = useState(0); @@ -273,7 +241,6 @@ export function ProjectsPage({ pdp, headerExtra, extraPanes }: ProjectExtras = { {open ? setOpen(null)}> { setOpen(null); reload(); }} onCancel={() => setOpen(null)} /> : null} diff --git a/web/src/apps/amazon-profit/ProjectsShared.tsx b/web/src/apps/amazon-profit/ProjectsShared.tsx index 04b47d54195a9a0bedaa911b747000dfa7bfe6b1..3f9955acdd816c662fa3c62fcc178785c63e0fd1 100644 --- a/web/src/apps/amazon-profit/ProjectsShared.tsx +++ b/web/src/apps/amazon-profit/ProjectsShared.tsx @@ -1,4 +1,4 @@ -import { createContext, useContext, useEffect, useId, useRef, useState, type KeyboardEvent, type ReactNode } from "react"; +import { createContext, useContext, useEffect, useId, useRef, useState, type ReactNode } from "react"; import { createPortal } from "react-dom"; import { CellSignalHigh } from "@phosphor-icons/react/dist/csr/CellSignalHigh"; import { CellSignalMedium } from "@phosphor-icons/react/dist/csr/CellSignalMedium"; @@ -35,7 +35,6 @@ const STATUS_GROUPS: Array<{ label: string; statuses: readonly Status[] }> = [ export interface Project { id: string; - type?: "project" | "pdp"; name: string; status: Status; priority: Priority; @@ -338,20 +337,6 @@ export function Owner({ name, compact = false }: { name: string | null; compact? // ---- The shared popup shell (S82-T90 reuses it for Competitors) ---- const ProjectFooterHost = createContext(null); -export function projectMenuKeys(event: KeyboardEvent) { - const items = Array.from(event.currentTarget.querySelectorAll('[role="menuitem"]:not(:disabled)')); - const index = items.indexOf(document.activeElement as HTMLElement); - const next = event.key === "ArrowDown" ? items[(index + 1) % items.length] - : event.key === "ArrowUp" ? items[(index - 1 + items.length) % items.length] - : event.key === "Home" ? items[0] : event.key === "End" ? items.at(-1) : undefined; - if (next) { event.preventDefault(); next.focus(); } -} - -const ProjectHeaderHost = createContext(null); -export function ProjectHeader({ children }: { children: ReactNode }) { - const host = useContext(ProjectHeaderHost); - return host ? createPortal(children, host) : null; -} export function ProjectFooter({ children }: { children: ReactNode }) { const host = useContext(ProjectFooterHost); return host ? createPortal(children, host) : <>{children}; @@ -372,7 +357,6 @@ export interface CenteredPopupProps { export function CenteredPopup({ title, subtitle, actions, onClose, children, className, size = "wide" }: CenteredPopupProps) { const panelRef = useRef(null); const [footer, setFooter] = useState(null); - const [header, setHeader] = useState(null); const titleId = useId(); useOverlayLayer({ panelRef, onDismiss: onClose, trapFocus: true, restoreFocus: true, initialFocus: "first", dismissOnOutside: false }); return @@ -388,14 +372,13 @@ export function CenteredPopup({ title, subtitle, actions, onClose, children, cla
{actions} - {className?.includes("lp-proj-popup") ? : null}
- +
{children}
{className?.includes("lp-proj-popup") ?
: null} - +
; diff --git a/web/src/apps/templateMarks.tsx b/web/src/apps/templateMarks.tsx index d9bd7200a0f32cb47ad8111951584729da19362c..902a0e9720ac785090b810fb0b1511c3d4e42b18 100644 --- a/web/src/apps/templateMarks.tsx +++ b/web/src/apps/templateMarks.tsx @@ -85,10 +85,6 @@ export const TEMPLATE_MARKS: Readonly> = { alerts: <> , - reports: <> - - - , }; const AMAZON_MARKS = new Set([ diff --git a/web/src/automation/AutomationDetail.tsx b/web/src/automation/AutomationDetail.tsx index 0354da555a1b3063db86a4c39af3f12e8e2e770c..6cd5d0c503bf5e985cc71bc997371dfe84451f06 100644 --- a/web/src/automation/AutomationDetail.tsx +++ b/web/src/automation/AutomationDetail.tsx @@ -22,7 +22,7 @@ import PanelToggleIcon from "../shell/PanelToggleIcon"; // what this codebase treats as a defect rather than a summary. // --------------------------------------------------------------------------- import { useCallback, useEffect, useMemo, useRef, useState } from "react"; -import { useRefusalToast } from "../shell/ToastStack"; +import { TOAST_EVENT } from "../apiContract"; import AutomationBuilder, { SavedQuerySchedules } from "./AutomationBuilder"; import HarnessPanel from "./HarnessPanel"; @@ -335,6 +335,28 @@ function RetiredAutomationResult({ automation }: { automation: Automation }) { ); } +function useRefusalToast() { + const [messages, setMessages] = useState<{ id: number; text: string }[]>([]); + const sequence = useRef(0); + const delivered = useRef(0); + useEffect(() => { + if (!messages.length) return; + const through = messages[messages.length - 1].id; + for (const message of messages) { + if (message.id <= delivered.current) continue; + delivered.current = message.id; + const text = message.text; + window.dispatchEvent(new CustomEvent(TOAST_EVENT, { detail: { text, tone: "error" } })); + } + setMessages((current) => current.filter((message) => message.id > through)); + }, [messages]); + return useCallback((text: string) => { + if (!text) return; + const message = { id: ++sequence.current, text }; + setMessages((current) => [...current, message]); + }, []); +} + export default function AutomationDetailView(props: Props) { return props.automation.retired ? diff --git a/web/src/automation/AutomationSurface.tsx b/web/src/automation/AutomationSurface.tsx index de5769baed09296fcebfba339dba6dfe908f2ee9..589786f0b725e5518b47f8d095e8373c86ea84f8 100644 --- a/web/src/automation/AutomationSurface.tsx +++ b/web/src/automation/AutomationSurface.tsx @@ -18,8 +18,7 @@ import type { KeyboardEvent as ReactKeyboardEvent } from "react"; import { Suspense, lazy, useCallback, useEffect, useRef, useState } from "react"; import type { AutomationOpenDetail } from "../apiContract"; -import { AUTOMATION_OPEN_EVENT, UNAUTHORIZED_EVENT, currentTenant } from "../apiContract"; -import { useRefusalToast } from "../shell/ToastStack"; +import { AUTOMATION_OPEN_EVENT, TOAST_EVENT, UNAUTHORIZED_EVENT, currentTenant } from "../apiContract"; // ⭐ WAVE 26 · ITEM 18 / R14 + C7 — THE LOOPABLE LOOP MARK, CONSUMED, NEVER REDRAWN. // The owner's reason is the mark's own geometry: *"these automations are basically loops."* // C7 says consume the existing icon component and do not inline a new SVG path, and @@ -212,6 +211,27 @@ if (typeof window !== "undefined") { * that a database can be created BY asking for an automation. */ +function useRefusalToast() { + const [messages, setMessages] = useState<{ id: number; text: string }[]>([]); + const sequence = useRef(0); + const delivered = useRef(0); + useEffect(() => { + if (!messages.length) return; + const through = messages[messages.length - 1].id; + for (const { id, text } of messages) { + if (id <= delivered.current) continue; + delivered.current = id; + window.dispatchEvent(new CustomEvent(TOAST_EVENT, { detail: { text, tone: "error" } })); + } + setMessages((current) => current.filter((message) => message.id > through)); + }, [messages]); + return (text: string) => { + if (!text) return; + const message = { id: ++sequence.current, text }; + setMessages((current) => [...current, message]); + }; +} + export default function AutomationSurface() { /* ⭐ T22 — SEEDED FROM THE CLIENT MEMO, so a revisit inside the freshness window never passes through `data === null` and therefore never paints the skeleton. The lazy initialiser runs diff --git a/web/src/connectors/ConnectorModal.tsx b/web/src/connectors/ConnectorModal.tsx index d988b126ba695e5d9def6d3ab513b382f7d79658..023287959c7e5d02890fa4828ccbbb90dd1f0b51 100644 --- a/web/src/connectors/ConnectorModal.tsx +++ b/web/src/connectors/ConnectorModal.tsx @@ -42,7 +42,6 @@ import type { DatabaseReveal, DatabasesTabHandle } from "./DatabasesTab"; import KeychainTab, { amazonRotationNotice, RotationNotice, useRotationClock } from "./KeychainTab"; import MonidRunDialog from "./MonidRunDialog"; import AmazonDeletionNotice from "./AmazonDeletionNotice"; -import AmazonWritesSwitch from "./AmazonWritesSwitch"; import { OdooSyncEvery } from "./OdooConfig"; import { AmazonSyncOverview, ConnectorSyncStatusLine, useConnectorSyncStatus, tileText } from "./ConnectorsPage"; import type { ConnectorRelationshipGraph, ConnectorRow } from "./ConnectorsPage"; @@ -713,10 +712,27 @@ export function ConnectorMark({ connectorKey, label }: { connectorKey: string; l ); } -/** Amazon Settings accepts the SP-API connector and the legacy review connector keys. */ +/** + * ⭐ THE OVERVIEW TAB, PURE AND EXPORTED — for the same reason `ConnectorCard` and + * `DirectoryGaps` are. The modal fetches on mount and `renderToStaticMarkup` does not run + * effects, so a static render of the modal is a spinner forever. This ticket's done-when is + * VISUAL, so the thing that has to be looked at has to be paintable from a payload. + */ +/** + * ⭐⭐ W49-T19 · R31 — IS THIS THE AMAZON CARD? Read off the KEY, never the label. + * + * `core/connector_directory.monid_rows` mints `md_` + `slug(name)`, so Amazon is `md_amazon` and + * the key is stable in a way a display string is not: a label is a sentence somebody may rewrite, + * and matching on one would silently stop offering the pull the day it changed. + * + * ⚠ THE NATIVE HALF IS INCLUDED ON PURPOSE. R27 keeps a catalogue card beside a native one for + * the four overlapping platforms, and if a native Amazon connector is ever added it reaches the + * same reviews through the same paid slice — so both keys open the same door rather than the + * feature quietly belonging to whichever row happened to exist first. + */ export function isAmazonConnector(key: string): boolean { const k = String(key || "").toLowerCase(); - return k === "md_amazon" || k === "amazon" || k === "amazon_sp"; + return k === "md_amazon" || k === "amazon"; } export function ConnectorOverviewPanel({ @@ -804,7 +820,6 @@ export function ConnectorOverviewPanel({ : null} {!hidden && connectorKey === "amazon_sp" ? : null} - {!hidden && isAmazonConnector(connectorKey) ? : null} {syncError ?

{syncError}

: null} {syncReady && !hidden && overview?.key === "amazon_sp" && isAdmin ? ( @@ -990,7 +1005,7 @@ export default function ConnectorModal({ const [pullSurface, setPullSurface] = useState(null); const [tabConfirmSurface, setTabConfirmSurface] = useState(null); const [leaveSurface, setLeaveSurface] = useState(null); - const canPull = isAmazonConnector(row.key) && row.key.toLowerCase() !== "amazon_sp"; + const canPull = isAmazonConnector(row.key); const unsavedMessage = databasePendingCount === 1 ? "You have 1 unsaved change to this connection." diff --git a/web/src/connectors/ConnectorsPage.tsx b/web/src/connectors/ConnectorsPage.tsx index bd9208f8e25bdc03cd8a701d7bf24b16febe715d..5db34460ac6977060512092e4dc45339ec08266c 100644 --- a/web/src/connectors/ConnectorsPage.tsx +++ b/web/src/connectors/ConnectorsPage.tsx @@ -354,8 +354,9 @@ export function AmazonSyncSchedule({ reading, problem = "", onRetry }: { data-amazon-table={row.table} data-connection-id={row.connectionId} data-table-sync-state={row.state}> {row.table.replace(/_/g, " ")} {multipleAccounts ? {row.account} : null} - {row.lastSyncAt - ? : "Not recorded"} + {row.state === "not_offered" ? "Not available" + : row.lastSyncAt ? + : "Not recorded"} {Date.parse(row.nextSyncAt) <= Date.now() + 60000 ? "Due now" : } diff --git a/web/src/customer-grid/CustomerGrid.tsx b/web/src/customer-grid/CustomerGrid.tsx index 99fa2060bab293af742ad96c62f6c9813e5c2332..2601c9ea8c961ee55b453acfc8dbf501a4184a30 100644 --- a/web/src/customer-grid/CustomerGrid.tsx +++ b/web/src/customer-grid/CustomerGrid.tsx @@ -84,7 +84,7 @@ import { cleanPercent, computeAggNumbers, computeAggs, formatServerAgg, formatSh import { SUMMARY_BAR_PX, SummaryBar } from "./summary/SummaryBar"; import type { CohortSets } from "./useVisibleRows"; import type { MeasureSets } from "./useVisibleRows"; -import { drawAppCell, useGetCellContent } from "./useGetCellContent"; +import { useGetCellContent } from "./useGetCellContent"; import { effectiveGridRows, savedValuePredicateNote } from "./effectiveGridRows"; import type { GroupRowPlaceholder } from "./useGetCellContent"; import { useGridSelection } from "./useGridSelection"; @@ -95,9 +95,8 @@ import { cellNoteText, editOutbox, gridEditScope, patchCellNote, pendingCellEdit import CellNoteMenu, { cellNoteRefusal, drawCellNoteMarker } from "./CellNoteMenu"; import type { OutboxEdit } from "./editOutbox"; import { LP_RED_DEEP, LP_YELLOW_DEEP } from "./theme"; -import type { AppTableHost } from "./appHostTypes"; +import type { AppTableHost } from "./seam"; import { DatePicker } from "./DatePicker"; -import { viewAsActive } from "../settings/viewAs"; /** * ⭐⭐ W51-T03 / OWNER RULING R6 + CONTRACT C8 — THE ONE CONFIRM DIALOG, for every destructive * act this grid can start: a field delete, a view delete and a record delete, plus the duplicate @@ -2107,7 +2106,6 @@ function hostCell(renderers: AppTableHost["cellRenderers"], field: Field): HostC if (!renderers) return {}; const chain = [renderers[field.key], renderers[`type:${field.type}`], renderers["*"]]; return { - draw: chain.find((entry) => entry?.draw)?.draw, progressColors: chain.find((entry) => entry?.progressColors)?.progressColors, button: chain.find((entry) => entry?.button !== undefined)?.button, activate: chain.find((entry) => entry?.activate)?.activate, @@ -2676,7 +2674,6 @@ function CustomerGridSurface({ const requestedEmbedded = embeddedProp && !hosted; // R8: a server-pinned related read keeps the same row/schema refusal as a projection. const embedded = requestedEmbedded || !!related || !!personalLayoutKey; - const viewAsReadOnly = viewAsActive(); const saveIncludes = useCallback((edit: OutboxEdit) => !embedded && gridEditScope(edit, scope), [embedded, scope]); const saveStatus = useSaveState(saveIncludes); const cellNoteDrafts = useMemo(() => new Map(pendingCellNotes(scope).map((edit) => @@ -2686,10 +2683,10 @@ function CustomerGridSurface({ return pending ? pending.text : cellNoteText(record, key); }, [cellNoteDrafts]); const emitHostEvent = useMemo(() => { - const emit = bindHostEvent(scope, !embedded && !viewAsReadOnly); + const emit = bindHostEvent(scope, !embedded); // C3: a hosted grid never moves the database's saved resume View, so it drops `view_select`. return hosted ? (event: HostEvent) => event.type !== "view_select" && emit(event) : emit; - }, [scope, embedded, hosted, viewAsReadOnly]); + }, [scope, embedded, hosted]); const personalLayout = usePersonalGridLayout(personalLayoutKey); const personalLayoutStamp = useRef(null); const [personalLayoutInitialized, setPersonalLayoutInitialized] = useState(null); @@ -2794,7 +2791,7 @@ function CustomerGridSurface({ // database's own saved views. includeWorkspace: hostWorkspace || !!related, // `writable: false` blocks `patchOverlay`, i.e. every cell edit. R2 wants them in Query. - writable: !embedded && !viewAsReadOnly, + writable: !embedded, /* ⭐⭐ W51-T06 / C6 / R5 — THE PINNED READ. Not a fourth flag beside the three above: those three answer "may this surface write / whose workspace is this / may cells be edited", and all three keep their normal answers here. What `related` changes is the READ. */ @@ -3885,7 +3882,7 @@ function CustomerGridSurface({ adoptApplication(application); }; adopt(local); - if (viewAsReadOnly || current.capabilities?.edit !== true || cleanDisplay(saved)?.mode !== picked.mode) return; + if (current.capabilities?.edit !== true || cleanDisplay(saved)?.mode !== picked.mode) return; const mine = ++applicationDisplaySeq.current; void fetch(`${API_V1}/applications/${encodeURIComponent(current.id)}`, { method: "PATCH", credentials: CREDENTIALS, headers: { "Content-Type": "application/json" }, @@ -3902,7 +3899,7 @@ function CustomerGridSurface({ adopt(current); signal(TOAST_EVENT, error instanceof Error ? error.message : "The Application could not save this layout."); }); - }, [resolvedApplication, adoptApplication, viewAsReadOnly]); + }, [resolvedApplication, adoptApplication]); const viewControlsReadOnly = !personalLayoutKey && viewControlsLocked( views.find((view) => view.id === activeViewId), payload?.viewer); @@ -3949,7 +3946,7 @@ function CustomerGridSurface({ setConfig(accepted); // Only cellView user updates report. Initialization and echoed cellFields use setConfig // directly, never updateConfig, so neither can emit or start a persistence feedback loop. - if (!viewAsReadOnly && cellView && onCellDisplayChange) { + if (cellView && onCellDisplayChange) { const visible = accepted.order.filter((key) => accepted.visible.includes(key)); const previous = config.order.filter((key) => config.visible.includes(key)); // C4 carries only mode and ordered visibility. Sorts and widths remain in this mount. @@ -3963,7 +3960,7 @@ function CustomerGridSurface({ } }, [config, activeViewId, views, viewControlsReadOnly, isQueryPreview, queryBinding, setFold, cellView, onCellDisplayChange, - hostOwnsDisplay, resolvedApplicationMatches, writeApplicationDisplay, viewAsReadOnly] + hostOwnsDisplay, resolvedApplicationMatches, writeApplicationDisplay] ); /** C2 returns the entire state so it can prove it did not rewrite user clauses. B consumes * exactly its managed addition/removal; filters, conjunction, user membership and every other @@ -3998,7 +3995,7 @@ function CustomerGridSurface({ onColumnProposeMove, setColumnVisible, insertColumn, - } = useGridColumns(fields, columnConfig, updateConfig, appHost); + } = useGridColumns(fields, columnConfig, updateConfig); // One resolved host cell per field. A Table grid resolves none. const hostRenderers = appHost?.cellRenderers; const hostCells = useMemo(() => { @@ -4084,7 +4081,7 @@ function CustomerGridSurface({ */ const isUserTable = !embedded && scope.startsWith("ut_"); const recordsMutable = payload?.recordsMutable !== false; - const canMutateRecords = isUserTable && recordsMutable && !viewAsReadOnly; + const canMutateRecords = isUserTable && recordsMutable; // C2: a host can withhold the New record row. Edit and delete stay. const canAppendRecords = canMutateRecords && appHost?.data.append !== false; const fieldBulkDoors = useFieldBulkDoors({ @@ -4294,7 +4291,7 @@ function CustomerGridSurface({ // R8's separate personal door. Saves are queued outside the mount so switching tabs cannot // drop a just-edited filter. This never modifies `views`, local workspace state, or host events. useEffect(() => { - if (viewAsReadOnly || !personalLayoutKey || !workspaceReady || !personalLayout.ready || personalLayoutInitialized !== storageKey || personalLayoutStamp.current === null) return; + if (!personalLayoutKey || !workspaceReady || !personalLayout.ready || personalLayoutInitialized !== storageKey || personalLayoutStamp.current === null) return; const layout = personalGridLayout(config); const stamp = JSON.stringify(layout); if (stamp === personalLayoutStamp.current) return; @@ -4305,10 +4302,10 @@ function CustomerGridSurface({ if (!saved) personalLayoutStamp.current = ""; }); }, [personalLayoutKey, workspaceReady, personalLayout.ready, personalLayout.save, config, - personalLayoutInitialized, storageKey, viewAsReadOnly]); + personalLayoutInitialized, storageKey]); useEffect(() => { - if (viewAsReadOnly || !workspaceReady || embedded || hostOwnsLayout) return; + if (!workspaceReady || embedded || hostOwnsLayout) return; const active = views.find((view) => view.id === activeViewId); if (!active || retiredHostViewIds.has(active.id) || !canSaveViewDisplay(active.config.display, config.display) || sameConfig(active.config, config)) return; @@ -4329,7 +4326,7 @@ function CustomerGridSurface({ void routed.result.then((result) => { if (!result.ok) signal(TOAST_EVENT, result.message); }); } }, [workspaceReady, activeViewId, config, views, embedded, queryBinding, scope, hostOwnsLayout, - emitHostEvent, retiredHostViewIds, sendNativeViewOps, viewAsReadOnly]); + emitHostEvent, retiredHostViewIds, sendNativeViewOps]); // Loading can replace the grid node without changing workspace readiness. // Observe the mounted node so the canvas also measures after that replacement. @@ -4434,7 +4431,6 @@ function CustomerGridSurface({ return () => controller.abort(); }, [routingPrefsAppId, workspaceReady, routingPrefsFieldsReady, routingPrefsSettled]); useEffect(() => { - if (viewAsReadOnly) return; if (!routingPrefsAppId || routingPrefsSettled !== routingPrefsAppId) return; if (hasLockedFilter(config.filters)) return; const signature = routeFilterSignatureOf(config.filters, config.filterConj); @@ -4447,7 +4443,7 @@ function CustomerGridSurface({ void writeRoutingFilters(appId, value); }, ROUTING_FILTERS_WRITE_DELAY_MS); return () => window.clearTimeout(timer); - }, [routingPrefsAppId, routingPrefsSettled, config.filters, config.filterConj, viewAsReadOnly]); + }, [routingPrefsAppId, routingPrefsSettled, config.filters, config.filterConj]); /** * ⭐⭐ D-673 / W54-T21 — THE TOP RUNG OF THE MAP'S ESCAPE LADDER, AND IT HAS TO BE HERE. * @@ -4689,11 +4685,11 @@ function CustomerGridSurface({ // door (cells, pickers, drawer): types.mayEditField — stratum + read-only-by-nature + // permissions vs the viewer, fail-closed on restricted fields when the viewer is unknown. const viewer = payload?.viewer; - const canAddField = !embedded && !workspacePending && !workspaceError && !viewAsReadOnly && + const canAddField = !embedded && !workspacePending && !workspaceError && (!isUserTable || (payload?.workspace as { canAddField?: boolean } | undefined)?.canAddField === true); const canEditField = useCallback( - (f: Field): boolean => !embedded && mayEditField(f, viewer) && !viewAsReadOnly, - [embedded, viewer, viewAsReadOnly] + (f: Field): boolean => !embedded && mayEditField(f, viewer), + [embedded, viewer] ); // Wave-5 items 9/11 — CLIENT-COMPUTED cells, injected AT their field keys so the whole @@ -5858,13 +5854,6 @@ function CustomerGridSurface({ if (row?.kind === "data" && cellSaveFailures.has(JSON.stringify([row.record.pid, key]))) drawCellNoteMarker(ctx, { ...rect, width: rect.width - (noteAt(row.record, key) ? 9 : 0) }, LP_RED_DEEP); }; - const hostDraw = hostCells.get(key)?.draw; - const field = fieldByKey.get(key); - if (hostDraw && field && row?.kind === "data") { - drawAppCell(args, hostDraw, { ...row.record, ...overlayEdits[row.record.pid] }, field); - markNote(); - return; - } const text = cell.kind === GridCellKind.Number || cell.kind === GridCellKind.Text ? cell.displayData : undefined; if (text === undefined || !text.includes(SHOW_AS_BREAK)) { drawContent(); markNote(); return; } @@ -5891,7 +5880,7 @@ function CustomerGridSurface({ ctx.fillText(share, x, mid + SHOW_AS_LINE_GAP); ctx.restore(); markNote(); - }, [displayRows, visibleCols, noteAt, cellSaveFailures, hostCells, fieldByKey, overlayEdits]); + }, [displayRows, visibleCols, noteAt, cellSaveFailures]); // W40-T30 — the wrapper that sat here intercepted `cell[0] === visibleCols.length` (the one // index past the real columns) to paint the virtual "Go" Uri cell. Owner instruction 27 // removed the feature, so every index glide asks for is a real column again and the hook's @@ -6038,7 +6027,7 @@ function CustomerGridSurface({ setServerFilteredSelection(null); }, [serverWindowed, windowPredicate]); const selectAllServerFiltered = useCallback(async () => { - if (viewAsReadOnly || !serverWindowed) return; + if (!serverWindowed) return; try { const response = await fetch( `${API_V1}/odoo-tables/${encodeURIComponent(scope)}/filtered-selection`, @@ -6072,7 +6061,7 @@ function CustomerGridSurface({ } catch { signal(TOAST_EVENT, "Could not select all filtered records. Try again."); } - }, [serverWindowed, scope, config.filters, config.filterConj, config.sorts, search, viewAsReadOnly]); + }, [serverWindowed, scope, config.filters, config.filterConj, config.sorts, search]); /* ════════════════════════ owner item 16 / R4 / C-UNDO ════════════════════════ THE RECORDING LAYER. `undoStack.ts` owns the stack and every inverse; this owns the one thing it cannot: reading the value a cell held BEFORE the write, which only exists at the @@ -6184,7 +6173,6 @@ function CustomerGridSurface({ /** ONE cell write, recorded. Every editor, picker and drag goes through this. */ const patchAndRecord = useCallback( (pid: number, updates: Partial, label = "an edit") => { - if (viewAsReadOnly) return; const reason = numberBatchRefusal(fields, [updates]); if (reason) { window.dispatchEvent(new CustomEvent(TOAST_EVENT, { detail: { text: reason, tone: "error" } })); @@ -6198,7 +6186,7 @@ function CustomerGridSurface({ patchOverlay(pid, updates); recordCells(changes, label); }, - [currentValue, patchOverlay, recordCells, fields, viewAsReadOnly] + [currentValue, patchOverlay, recordCells, fields] ); const [chatOpen, setChatOpen] = useState(false); @@ -6227,7 +6215,6 @@ function CustomerGridSurface({ /** MANY cells, ONE stack entry — a paste and a bulk clear are each one user action (R4). */ const patchManyAndRecord = useCallback( (writes: { pid: number; updates: Partial }[], label: string) => { - if (viewAsReadOnly) return; const byPid = new Map>(); for (const { pid, updates } of writes) byPid.set(pid, { ...byPid.get(pid), ...updates }); @@ -6252,17 +6239,17 @@ function CustomerGridSurface({ for (const w of writes) patchOverlay(w.pid, w.updates); recordCells(changes, label); }, - [currentValue, patchOverlay, recordCells, fields, viewAsReadOnly] + [currentValue, patchOverlay, recordCells, fields] ); const bulkEditableFields = useMemo( - () => fields.filter((field) => !viewAsReadOnly && ( + () => fields.filter((field) => ( serverWindowed // Connected records themselves are source-owned. Only an already-declared shared // overlay field is writable through the server descriptor, with the usual field grant. ? field.shared === true && mayEditField(field, viewer) : canEditField(field) ) && field.type !== "rating"), - [fields, serverWindowed, viewer, canEditField, viewAsReadOnly] + [fields, serverWindowed, viewer, canEditField] ); const bulkField = bulkEditableFields.find((field) => field.key === bulkFieldKey) ?? bulkEditableFields[0]; const bulkSelectedCount = serverFilteredSelection?.matched ?? selectedPids.size; @@ -6325,7 +6312,6 @@ function CustomerGridSurface({ */ const applyEntry = useCallback( async (entry: UndoEntry, dir: "back" | "forward") => { - if (viewAsReadOnly) return; const op = directed(entry, dir); if (op.kind === "cells") { const byPid = new Map>(); @@ -6411,7 +6397,7 @@ function CustomerGridSurface({ } signal(TOAST_EVENT, describe(entry, dir)); }, - [patchOverlay, withdrawPending, emitHostEvent, fields, acceptPendingRow, mintRid, setOverlayEdits, viewAsReadOnly] + [patchOverlay, withdrawPending, emitHostEvent, fields, acceptPendingRow, mintRid, setOverlayEdits] ); /** @@ -6445,7 +6431,6 @@ function CustomerGridSurface({ if (!(event.ctrlKey || event.metaKey) || event.altKey) return; const key = event.key.toLowerCase(); if (key !== "z" && key !== "y" && key !== "f") return; - if (viewAsReadOnly && key !== "f") return; const el = document.activeElement as HTMLElement | null; if (el && (el.tagName === "INPUT" || el.tagName === "TEXTAREA" || el.isContentEditable)) return; @@ -6495,7 +6480,7 @@ function CustomerGridSurface({ }; window.addEventListener("keydown", onKey, true); return () => window.removeEventListener("keydown", onKey, true); - }, [scope, applyEntry, applicationHost.paged, embedded, queryBinding, hostOwnsLayout, activeViewId, viewAsReadOnly]); + }, [scope, applyEntry, applicationHost.paged, embedded, queryBinding, hostOwnsLayout, activeViewId]); /** @@ -6957,12 +6942,9 @@ function CustomerGridSurface({ if (vr?.kind === "data" && cellCol) { const field = fieldByKey.get(cellCol.id!); const text = field ? formatDisplay(field, vr.record[field.key]) : ""; - const draw = field ? hostCells.get(field.key)?.draw : undefined; - const reason = draw?.unavailable && vr.record[draw.unavailable.reasonKey]; const bounds = args.bounds; const tip = field && (noteAt(vr.record, field.key) || cellSaveFailures.has(JSON.stringify([vr.record.pid, field.key]))) ? null - : typeof reason === "string" && reason ? reason : bounds ? cellTipText(text, measureCellText(text), bounds.width) : null; if (tip && bounds) { const x = tipLeft(bounds.x, window.innerWidth, measureCellText(tip) + 20); @@ -6978,7 +6960,7 @@ function CustomerGridSurface({ } setCellTip((prev) => (prev === null ? prev : null)); }, - [visibleCols, fieldByKey, displayRows, noteAt, scope, cellSaveFailures, viewColumnFailures, hostCells] + [visibleCols, fieldByKey, displayRows, noteAt, scope, cellSaveFailures, viewColumnFailures] ); const rowHeight = useMemo( () => @@ -7567,7 +7549,6 @@ function CustomerGridSurface({ const openHeaderMenu = useCallback( (column: number, bounds: Rectangle) => { - if (viewAsReadOnly) return; const definition = visibleCols[column]; if (!definition?.id) return; if (embedded) gridRef.current?.focus(); @@ -7583,7 +7564,7 @@ function CustomerGridSurface({ }, }); }, - [embedded, visibleCols, viewAsReadOnly] + [embedded, visibleCols] ); // W40-T30 — the resize wrapper here existed for one reason: to swallow a drag on the virtual // "Go" column, which had no field to persist a width to. With that column gone (owner @@ -7874,7 +7855,7 @@ function CustomerGridSurface({ * holds exactly one view), so Duplicate cannot smuggle a second one in. */ const persistView = useCallback((view: SavedView, write?: { action?: string }) => { - if (viewAsReadOnly || embedded || personalLayoutKey || hostOwnsLayout) return; + if (embedded || personalLayoutKey || hostOwnsLayout) return; // Refuse even a caller that stripped the retirement marker from its incoming draft. const storedDisplay = views.find((item) => item.id === view.id)?.config.display; if (retiredHostViewIds.has(view.id) || !canSaveViewDisplay(storedDisplay, view.config.display)) return; @@ -7904,7 +7885,7 @@ function CustomerGridSurface({ else if (routed.channel === "query") void routed.result.then((result) => { if (!result.ok) signal(TOAST_EVENT, result.message); }); }, [queryBinding, scope, hostOwnsLayout, embedded, personalLayoutKey, emitHostEvent, views, - retiredHostViewIds, activeViewId, sendNativeViewOps, viewAsReadOnly]); + retiredHostViewIds, activeViewId, sendNativeViewOps]); /** * Item 12 (C-LOCK) — set or clear a view's cohort lock, from the rail's menu. @@ -9309,7 +9290,6 @@ function CustomerGridSurface({ */ const prepareSelectedViews = useCallback( async (viewIds: string[]) => { - if (viewAsReadOnly) return; const ids = [...new Set(viewIds)]; const key = exportSelectionKey(ids); const current = xlsxPrepRef.current; @@ -9426,7 +9406,7 @@ function CustomerGridSurface({ fail("Selected Views could not be exported."); } }, - [views, scope, embedded, isQueryPreview, fields, fieldByKey, measureSets, cohortSets, today, viewAsReadOnly] + [views, scope, embedded, isQueryPreview, fields, fieldByKey, measureSets, cohortSets, today] ); /** @@ -9546,7 +9526,6 @@ function CustomerGridSurface({ }, [scope, activeViewId, emitHostEvent]); const saveField = useCallback( (field: Field, scope?: FieldScope) => { - if (viewAsReadOnly) return Promise.resolve(false); const requestId = eventId("field"); const writeScope = workspaceScopeRef.current; const isNew = !fields.some((item) => item.key === field.key); @@ -9583,7 +9562,7 @@ function CustomerGridSurface({ return saved; }); }, - [stampFieldEdit, fields, emitHostEvent, viewAsReadOnly] + [stampFieldEdit, fields, emitHostEvent] ); const createField = useCallback( ( @@ -9820,7 +9799,7 @@ function CustomerGridSurface({ */ const setProfileFlag = useCallback( (key: string, on: boolean) => { - if (viewAsReadOnly || !isUserTable) return; + if (!isUserTable) return; void patchTableField(scope, key, { profile: on ? { source: "instagram" } : null, }).then((made) => { @@ -9829,7 +9808,7 @@ function CustomerGridSurface({ signal(WORKSPACE_STALE_EVENT); }); }, - [isUserTable, scope, viewAsReadOnly] + [isUserTable, scope] ); /** @@ -9856,7 +9835,7 @@ function CustomerGridSurface({ shared column the overlay stratum would fork it, and this one edits it in place. */ showAs?: ShowAs; }) => { - if (viewAsReadOnly || !isUserTable) return; + if (!isUserTable) return; void patchTableField(scope, key, patch as Record).then((made) => { if (!made) return; // the bridge has already said why setFields((current) => current.map((f) => (f.key === made.key ? made : f))); @@ -9867,21 +9846,21 @@ function CustomerGridSurface({ signal(ROWS_STALE_EVENT); }); }, - [isUserTable, scope, viewAsReadOnly] + [isUserTable, scope] ); // C3 owns both the state and the wall. `session.admin` is its current Owner mapping; // this client gate only avoids advertising a PATCH the server will refuse. const setDeletionLock = useCallback( (key: string, deletionLocked: boolean) => { - if (viewAsReadOnly || !isUserTable || !viewer?.isAdmin) return; + if (!isUserTable || !viewer?.isAdmin) return; void patchTableField(scope, key, { deletionLocked }).then((made) => { if (!made) return; setFields((current) => current.map((f) => (f.key === made.key ? made : f))); signal(WORKSPACE_STALE_EVENT); }); }, - [isUserTable, scope, viewer?.isAdmin, viewAsReadOnly] + [isUserTable, scope, viewer?.isAdmin] ); /** @@ -10766,7 +10745,7 @@ function CustomerGridSurface({ leads to a pane with no editor in it. On a `ut_*` kanban (every automation board) the old `custom` test was false for every column, so the lane header offered nothing. */ const kanbanCanAddOption = - !viewAsReadOnly && !!kanbanField && kanbanField.type === "select" && + !!kanbanField && kanbanField.type === "select" && isUserSchemaField(kanbanField, isUserTable); const onKanbanAddOption = useCallback( (anchor: AnchorRect) => { @@ -11044,7 +11023,7 @@ function CustomerGridSurface({ // C2, R29: a host's one toolbar slot takes the mode switch's place, and History sits beside it. const toolbarModeSlot = appHost ? <> {appHost.toolbar} - {appHost.history !== false && isUserTable && } : modeControl; @@ -11072,7 +11051,7 @@ function CustomerGridSurface({ ...linkAdd.added, ])]; const linkSeed = linkField?.type === "link" ? linkSeedOf(linkField) : undefined; - const linkCanAdd = !viewAsReadOnly && !!linkSeed && !!linkTargetScope && linkTargetScope.startsWith("ut_") + const linkCanAdd = !!linkSeed && !!linkTargetScope && linkTargetScope.startsWith("ut_") && linkAdd.mutable; const addLinkedSeedRow = async () => { if (!linkAt || !linkSeed || !linkTargetScope || linkAdd.busy) return; @@ -11382,7 +11361,7 @@ function CustomerGridSurface({ fieldBulkDoors.onDeleteFields([key])} - onEditField={embedded || viewAsReadOnly ? undefined : (fieldKey, rect) => { + onDeleteField={embedded ? undefined : (key) => fieldBulkDoors.onDeleteFields([key])} + onEditField={embedded ? undefined : (fieldKey, rect) => { setColumnMenu({ fieldKey, pane: "edit", @@ -11440,14 +11419,13 @@ function CustomerGridSurface({ }, }); }} - onAddField={embedded || viewAsReadOnly ? undefined : (rect) => setPlusMenu({ left: rect.left, top: rect.top, right: rect.right, bottom: rect.bottom, width: rect.width, height: rect.height })} + onAddField={embedded ? undefined : (rect) => setPlusMenu({ left: rect.left, top: rect.top, right: rect.right, bottom: rect.bottom, width: rect.width, height: rect.height })} rowHeightMode={appHost ? appHost.columns?.rowHeightMode ?? "short" : config.rowHeightMode} rowHeightPx={appHost ? rowPx : undefined} // C2, R2: a host keeps its own height pick, so an App never writes the Table view's height. onRowHeightMode={(rowHeightMode) => appHost ? appHost.columns?.onRowHeightModeChange?.(rowHeightMode) : updateConfig({ ...config, rowHeightMode })} filters={config.filters} - appHost={appHost} onFilters={applyFilters} filterConj={config.filterConj ?? "and"} onFilterConj={(filterConj) => { @@ -11618,7 +11596,7 @@ function CustomerGridSurface({ {!hideViews && ( view.id)} @@ -12038,7 +12016,7 @@ function CustomerGridSurface({ ))} ; })} - + ; })} )}} @@ -12646,7 +12624,7 @@ function CustomerGridSurface({ docs={payload?.docs} docPayload={payload?.docPayload} onDocAdd={ - !embedded && payload?.docs && !viewAsReadOnly + !embedded && payload?.docs ? (pid, file) => emitHostEvent({ id: eventId("docadd"), @@ -12658,13 +12636,13 @@ function CustomerGridSurface({ : undefined } onDocFetch={ - !embedded && payload?.docs && !viewAsReadOnly + !embedded && payload?.docs ? (pid, docId) => emitHostEvent({ id: eventId("docget"), type: "doc_fetch", pid, docId }) : undefined } onDocDelete={ - !embedded && payload?.docs && !viewAsReadOnly + !embedded && payload?.docs ? (pid, docId) => emitHostEvent({ id: eventId("docdel"), type: "doc_delete", pid, docId }) : undefined @@ -12908,7 +12886,6 @@ function CustomerGridSurface({ height: starHover.size, }} aria-pressed={recordStars.starred(starHover.pid)} - disabled={viewAsReadOnly} title={recordStars.starred(starHover.pid) ? "Unstar this record" : "Star this record"} aria-label={ recordStars.starred(starHover.pid) ? "Unstar this record" : "Star this record" @@ -12988,7 +12965,6 @@ function CustomerGridSurface({ ))} - {appHost?.filterPanel ? appHost.filterPanel({ fields, filters, filterConj, - onFilters: (next) => { if (!viewControlsReadOnly && !filtersLocked) onFilters(next); }, - onFilterConj: (next) => { if (!viewControlsReadOnly && !filtersLocked) onFilterConj(next); }, - disabled: viewControlsReadOnly || filtersLocked, - }) : ) : undefined } - />} + /> )} @@ -1120,10 +1113,6 @@ export default function Toolbar({ ) : (
- {appHost?.sortChoices?.map((choice) => )} limitDetailCopy(limit.cause, keys)).filter(Boolean))]; const fixes = [...new Set(entries.map((limit) => limitDetailCopy(limit.recommendation, keys)).filter(Boolean))]; const grouped = { subject, effect, cause: causes[0] ?? "", recommendation: "" }; - short.push(gridLimitCopy(effect === "source_empty" - ? [...causes, ...fixes].join(" ") : plainLimit(grouped, counts, keys.length, calculated))); + short.push(gridLimitCopy(plainLimit(grouped, counts, keys.length, calculated))); const heading = effect === "window_clamped" ? "Paging" : effect === "slow" ? "Row loading" : effect === "unresolved" ? "Row matching" - : effect === "not_ready" || effect === "source_empty" ? "Data" + : effect === "not_ready" ? "Data" : subject; full.push(`${heading}: ${[...causes, ...fixes].join("\n")}`); } diff --git a/web/src/customer-grid/seam.ts b/web/src/customer-grid/seam.ts index 3f48b08da8067eced8d04fcacc50dc76ce6fd264..fcaaf29b6ad967e84b18393238489eec0749fe7a 100644 --- a/web/src/customer-grid/seam.ts +++ b/web/src/customer-grid/seam.ts @@ -1,14 +1,47 @@ import { createElement, lazy, Suspense, useEffect, useMemo, useState } from "react"; -import type { ComponentProps, ComponentType } from "react"; +import type { ComponentProps, ComponentType, ReactNode } from "react"; import { ACCESS_REFUSED_MESSAGE, fetchWorkspace } from "./apiBridge"; import type { SurfaceScope, WorkspaceStatus } from "./apiBridge"; -import type { RowHeightMode } from "./types"; -import type { AppTableHost } from "./appHostTypes"; -export type { AppColumns, AppTrendPoint, AppCellDraw, AppFilterPanelProps, AppSortChoice, - AppCellRenderer, AppTableHost } from "./appHostTypes"; +import type { Field, Row, RowHeightMode } from "./types"; import { LogoLoader } from "../ui/LogoLoader"; import "./amazonPeriodPicker.css"; +interface AppData { + database: string; + refusal?: string | null; + /** False withholds the New record row. Edit and delete keep their own rules. */ + append?: boolean; + onOpenRecord?: (pid: number, table: string, record?: Row, fields?: Field[]) => void; +} + +interface AppColumns { + widths?: Readonly>; + /** The row height while the host's own pick is unset. A view's stored height never applies. */ + rowHeight?: number; + /** The host's own height pick. AppTableGrid keeps it per reader and database. */ + rowHeightMode?: RowHeightMode; + onRowHeightModeChange?: (mode: RowHeightMode) => void; +} + +interface AppCellRenderer { + /** Progress fill colours. Absent, a progress cell keeps the Table paint. */ + progressColors?: { readonly inProgress: string; readonly completed: string }; + /** The cell draws as a button, and a click or Enter calls `activate`. */ + button?: boolean; + /** Runs before the grid's own picker, link window or editor. False hands the click back. */ + activate?: (row: Row, field: Field) => boolean | void; +} + +// Apps declare these four members. Table owns its rail, menus and view actions. +export interface AppTableHost { + data: AppData; + columns?: AppColumns; + /** Each member reads the field key, then `type:`, then `*`. */ + cellRenderers?: Readonly>; + /** Drawn in the toolbar's mode slot, beside the grid's History. */ + toolbar?: ReactNode; +} + // What a host takes over from the Table grid. The grid reads these, never the kind of host. export interface GridHostCapabilities { /** The host owns the layout: the grid adopts, selects, undoes and saves no View. */ @@ -62,15 +95,14 @@ export function AppTableGrid(appHost: AppTableHost) { }, [database, refusal]); const pickedMode = heightPick?.key === heightKey ? heightPick.mode : undefined; - const { data, columns: hostColumns, cellRenderers, toolbar, history, filterPanel, sortChoices } = appHost; - const host = useMemo(() => ({ data, cellRenderers, toolbar, history, filterPanel, sortChoices, - columns: { ...hostColumns, + const { data, columns: hostColumns, cellRenderers, toolbar } = appHost; + const host = useMemo(() => ({ data, cellRenderers, toolbar, columns: { ...hostColumns, rowHeightMode: hostColumns?.rowHeightMode ?? pickedMode, onRowHeightModeChange: (mode) => { setHeightPick({ key: heightKey, mode }); try { localStorage.setItem(heightKey, mode); } catch { /* The pick lasts for this mount. */ } hostColumns?.onRowHeightModeChange?.(mode); - } } }), [data, cellRenderers, toolbar, history, filterPanel, sortChoices, hostColumns, pickedMode, heightKey]); + } } }), [data, cellRenderers, toolbar, hostColumns, pickedMode, heightKey]); const current = admission?.database === database ? admission : null; const message = refusal || current?.refusal; diff --git a/web/src/customer-grid/useGetCellContent.ts b/web/src/customer-grid/useGetCellContent.ts index ad90950943212a3b63653aef7c8c251fd74963d7..e04579f8dcaae483a2b7b7b0271a7b85478847ea 100644 --- a/web/src/customer-grid/useGetCellContent.ts +++ b/web/src/customer-grid/useGetCellContent.ts @@ -15,142 +15,13 @@ import { useCallback, useMemo } from "react"; import { GridCellKind } from "@glideapps/glide-data-grid"; -import type { DrawCellCallback, GridCell, GridColumn, Item } from "@glideapps/glide-data-grid"; +import type { GridCell, GridColumn, Item } from "@glideapps/glide-data-grid"; import { TOTAL_GROUP_KEY } from "./types"; import type { Field, Row, VisibleRow } from "./types"; import type { OverlayEdits } from "./useCustomerData"; import { makeCell } from "./cells"; import { effectiveCellValue } from "./effectiveGridRows"; import { fitGroupLabel, GROUP_HEADER_FONT, GROUP_LABEL_PAD } from "./overlayPlacement"; -import type { AppCellDraw, AppTrendPoint } from "./appHostTypes"; -import { jsonParse, numericOrUndefined } from "./display"; -import { LP_GREEN, LP_GREEN_DEEP, LP_GREEN_TINT, LP_PURPLE, LP_PURPLE_DEEP, LP_RED, LP_RED_DEEP, LP_RED_TINT } from "./theme"; - -type AppDrawArgs = Parameters[0]; - -export function appTrendSeries(value: Row[string]): AppTrendPoint[] { - const parsed = jsonParse(value).value; - if (!Array.isArray(parsed)) return []; - return parsed.filter((point): point is AppTrendPoint => point !== null && typeof point === "object" - && typeof point.month === "string" && /^\d{4}-(0[1-9]|1[0-2])$/.test(point.month) - && (point.value === null || typeof point.value === "number" && Number.isFinite(point.value))) - .sort((a, b) => a.month.localeCompare(b.month)); -} - -function appScoreTone(value: number | undefined, max: number, args: AppDrawArgs) { - return value === undefined || value > max * .4 && value < max * .6 - ? { text: args.theme.textLight, fill: args.theme.bgBubble } - : value >= max * .6 ? { text: LP_GREEN_DEEP, fill: LP_GREEN_TINT } - : { text: LP_RED_DEEP, fill: LP_RED_TINT }; -} - -function drawAppScore(args: AppDrawArgs, value: number | undefined, max: number, ring: boolean, - draw: Extract) { - const { ctx, rect, theme } = args; - let tone = appScoreTone(value, max, args); - if (draw.kind === "score" && draw.neutralRange && value !== undefined) { - const [low, high] = draw.neutralRange; - const [floor, ceiling] = draw.outerRange ?? [-Infinity, Infinity]; - tone = { text: theme.textDark, fill: value < floor ? LP_RED : value < low ? LP_RED_TINT - : value <= high ? theme.bgBubble : value <= ceiling ? LP_GREEN_TINT : LP_GREEN }; - } - const x = rect.x + theme.cellHorizontalPadding + 15, y = rect.y + rect.height / 2; - ctx.strokeStyle = theme.borderColor; - ctx.lineWidth = 3; - if (ring && value !== undefined) { - ctx.beginPath(); ctx.arc(x, y, 12, 0, Math.PI * 2); ctx.stroke(); - ctx.strokeStyle = tone.text; - ctx.beginPath(); ctx.arc(x, y, 12, -Math.PI / 2, - -Math.PI / 2 + Math.PI * 2 * Math.max(0, Math.min(1, value / max))); ctx.stroke(); - } else if (!ring) { - ctx.fillStyle = tone.fill; - ctx.beginPath(); ctx.roundRect(x - 15, y - 12, 44, 24, 5); ctx.fill(); - } - const after = draw.kind === "ring" && draw.labelPosition === "after"; - ctx.fillStyle = after ? theme.textDark : tone.text; - ctx.textAlign = after ? "left" : "center"; - ctx.fillText(value === undefined ? draw.emptyText ?? "None" : ring ? String(after ? value : Math.round(value)) : value.toFixed(1), - after ? x + 22 : draw.kind === "score" && draw.neutralRange ? x + 7 : x, y); -} - -function drawAppTrend(args: AppDrawArgs, draw: Extract, row: Row) { - const { ctx, rect, theme } = args; - const points = appTrendSeries(row[draw.seriesKey]); - const values = points.flatMap((point) => point.value === null ? [] : [point.value]); - const max = Math.max(1e-12, ...values); - const value = numericOrUndefined(row[draw.valueKey]); - const change = numericOrUndefined(row[draw.changeKey]); - const number = new Intl.NumberFormat("en-US", { maximumFractionDigits: 1 }); - const changeText = change === undefined ? "N/A" : draw.changeDecimals === undefined ? number.format(change) - : draw.changeDecimals === 0 ? String(Math.round(change)) - : new Intl.NumberFormat("en-US", { maximumFractionDigits: draw.changeDecimals }).format(change); - const changeLabel = change === undefined ? changeText : `${change > 0 ? "+" : ""}${changeText}%`; - let label = value === undefined ? "N/A" : number.format(value) + (draw.format === "percent" ? "%" : ""); - if (value !== undefined && draw.valuePrecision === "adaptive" && draw.format === "percent") { - label = value === 0 ? "0%" : value < .001 ? "under 0.001%" - : value.toFixed(value < .01 ? 3 : value < 1 ? 2 : value < 10 ? 1 : 0) + "%"; - } - if (draw.showValue === false || draw.omitMissingValue && value === undefined) label = ""; - const labelWidth = (label ? ctx.measureText(label).width + 8 : 0) + ctx.measureText(changeLabel).width + 8; - const width = draw.fitLabels ? Math.max(0, Math.min(points.length * 10, - rect.width - theme.cellHorizontalPadding * 2 - labelWidth)) : Math.min(96, rect.width * .42); - const x = rect.x + theme.cellHorizontalPadding; - const y = rect.y + rect.height / 2, step = width / Math.max(1, points.length); - let last = -1; - points.forEach((point, index) => { if (point.value !== null) last = index; }); - points.forEach((point, index) => { - if (point.value === null) return; - const height = Math.max(1, 18 * Math.max(0, point.value) / max); - ctx.fillStyle = draw.barTone === "primary" ? index === last ? LP_PURPLE_DEEP : LP_PURPLE - : index === last ? theme.accentColor : theme.textLight; - ctx.fillRect(x + index * step, y + 9 - height, Math.max(draw.fitLabels ? .5 : 1, step - 3), height); - }); - if (label) { - ctx.fillStyle = theme.textDark; - ctx.fillText(label, x + width + 8, y); - } - ctx.textAlign = "right"; - const band = draw.changeNeutralBand ?? 0; - ctx.fillStyle = change === undefined || Math.abs(change) <= band ? theme.textLight - : change > 0 ? LP_GREEN_DEEP : LP_RED_DEEP; - ctx.fillText(changeLabel, - rect.x + rect.width - theme.cellHorizontalPadding, y); -} - -function drawAppBadge(args: AppDrawArgs, draw: Extract, row: Row, field: Field) { - const { ctx, rect, theme } = args; - const x = rect.x + theme.cellHorizontalPadding, y = rect.y + rect.height / 2; - const sub = draw.subKey ? String(row[draw.subKey] ?? "") : ""; - ctx.fillStyle = theme.textDark; - const label = String(row[field.key] ?? draw.emptyText ?? ""); - ctx.fillText(label, x, y - (sub ? 8 : 0)); - if (sub) { ctx.fillStyle = theme.textLight; ctx.fillText(sub, x, y + 9); } - const count = draw.countKey ? numericOrUndefined(row[draw.countKey]) : undefined; - if (count !== undefined && count > 0) { - const text = `+${Math.floor(count)}`, width = ctx.measureText(text).width + 12; - const chipX = Math.min(x + ctx.measureText(label).width + 8, rect.x + rect.width - width - 8); - ctx.fillStyle = theme.bgBubble; - ctx.beginPath(); ctx.roundRect(chipX, y - 11, width, 22, 5); ctx.fill(); - ctx.fillStyle = theme.textDark; ctx.fillText(text, chipX + 6, y); - } -} - -export function drawAppCell(args: AppDrawArgs, draw: AppCellDraw, row: Row, field: Field): void { - const { ctx, rect, theme } = args; - ctx.save(); - ctx.beginPath(); ctx.rect(rect.x + 1, rect.y + 1, rect.width - 2, rect.height - 2); ctx.clip(); - ctx.font = `${theme.baseFontStyle} ${theme.fontFamily}`; - ctx.textBaseline = "middle"; - ctx.textAlign = "left"; - if (draw.unavailable && row[draw.unavailable.reasonKey] === draw.unavailable.equals) { - ctx.fillStyle = theme.textLight; - ctx.fillText(draw.unavailable.text, rect.x + theme.cellHorizontalPadding, rect.y + rect.height / 2); - } else if (draw.kind === "trend") drawAppTrend(args, draw, row); - else if (draw.kind === "badge") drawAppBadge(args, draw, row, field); - else drawAppScore(args, numericOrUndefined(row[draw.valueKey ?? field.key]), - draw.max && draw.max > 0 ? draw.max : draw.kind === "ring" ? 100 : 10, draw.kind === "ring", draw); - ctx.restore(); -} const EMPTY_CELL: GridCell = { kind: GridCellKind.Text, diff --git a/web/src/customer-grid/useGridColumns.ts b/web/src/customer-grid/useGridColumns.ts index 62de41f3cc2da8fa42e065567e8783ea715e7027..e1cbb1dff1c147ca20e975d515547d48ea1290a4 100644 --- a/web/src/customer-grid/useGridColumns.ts +++ b/web/src/customer-grid/useGridColumns.ts @@ -8,7 +8,6 @@ import { fitHeaderTitle, hasShareMark, HEADER_MENU_SLOT, headerLabelSpace, headerMarkSizes } from "./overlayPlacement"; import { COLUMN_TONE_THEME, INVOLVED_HEADER_FONT, lightTheme } from "./theme"; import type { ControlTone } from "./theme"; -import type { AppTableHost } from "./appHostTypes"; const DEFAULT_WIDTH = 150; @@ -161,15 +160,6 @@ function columnTheme(tone: ControlTone | undefined): Partial | undefined return tone ? COLUMN_TONE_THEME[tone] : undefined; } -export function appHeaderTheme(tone: "primary" | undefined): Partial | undefined { - if (!tone) return undefined; - const style = getComputedStyle(document.documentElement); - const tint = style.getPropertyValue("--lp-primary-tint").trim(); - const text = style.getPropertyValue("--lp-primary").trim(); - return { bgHeader: tint, bgHeaderHovered: tint, bgHeaderHasFocus: tint, textHeader: text, - textHeaderSelected: text }; -} - /** * Owner item 22 — key → which control involves it, or absent. * @@ -351,8 +341,7 @@ export interface GridColumnsApi { export function useGridColumns( fields: Field[], config: ViewConfig, - onConfig: (next: ViewConfig) => void, - appHost?: AppTableHost + onConfig: (next: ViewConfig) => void ): GridColumnsApi { const insertionRef = useRef({ fields, config, onConfig }); insertionRef.current = { fields, config, onConfig }; @@ -430,10 +419,10 @@ export function useGridColumns( // column now, and all three tint it the same faint warm grey; the hue that says // WHICH control lives on the toolbar chip. User-created ownership moved out of the // theme entirely and is the dot drawn in the mark strip (R11). - themeOverride: appHeaderTheme(appHost?.columns?.headerTone?.[key]) ?? columnTheme(tones.get(key)), + themeOverride: columnTheme(tones.get(key)), }; }), - [order, visible, config.widths, fieldByKey, tones, appHost] + [order, visible, config.widths, fieldByKey, tones] ); const onColumnResize = useCallback( diff --git a/web/src/index.css b/web/src/index.css index 1bf3c6c99842489301b7fd3dd61909d0d372aabc..6e48c69177b34d77173ae9102c29e1a55c899d52 100644 --- a/web/src/index.css +++ b/web/src/index.css @@ -13678,12 +13678,6 @@ input.conn-odoo-v:disabled { background: var(--lp-surface-2); color: var(--lp-mu .cg-overlay.cg-cell-note-pop, .cg-overlay.cg-w46-stop-addmenu, .cg-overlay.home-name-tip, -.cg-overlay.lp-proj-status-menu, -.cg-overlay.pdp-brief-type-menu, -.cg-overlay.pdp-continue-menu, -.cg-overlay.pdp-option-menu, -.cg-overlay.pdp-settings-menu, -.cg-overlay.pdp-setup-pop, .cg-overlay.sheet-cell-export-pop, .cg-overlay.sheet-cell-menu-pop, .cg-overlay.sheet-chart-color-list, diff --git a/web/src/settings/ActivityLogPage.tsx b/web/src/settings/ActivityLogPage.tsx index 4c9ef67cd5b937a6ecfbe2e924b03460254b25a7..b55fa458217bc445f2a6e23633054b66e239a173 100644 --- a/web/src/settings/ActivityLogPage.tsx +++ b/web/src/settings/ActivityLogPage.tsx @@ -3,10 +3,9 @@ import { ArrowClockwise, ArrowsLeftRight, CaretRight, CursorClick, Funnel, Globe import { API_V1, CREDENTIALS, checkTenant, UNAUTHORIZED_EVENT } from "../apiContract"; import { isAdmin } from "../shell/session"; import type { SessionUser } from "../shell/session"; -import { FilterBuilderPanel, Popover, countConditions, createFilterRule, opsForType } from "../filter-kit"; -import type { Field, FilterTree } from "../filter-kit"; +import { FilterBuilderPanel, Popover, countConditions } from "../filter-kit"; import { LockMark } from "../customer-grid/icons"; -import { DEFAULT_FILTERS, activityDescription, activityFieldConditionCount, activityFilterFields, activityTargetName, filterActivity, groupActivityByDay } from "./activityLog"; +import { DEFAULT_FILTERS, activityDescription, activityFilterFields, activityTargetName, filterActivity, groupActivityByDay } from "./activityLog"; import type { ActivityEntry, ActivityEvent, ActivityFilters, ActivityLimits, MovementSample } from "./activityLog"; import "./activityLog.css"; @@ -118,48 +117,6 @@ function EventRow({ entry }: { entry: ActivityEntry }) { ; } -function ActivityFilterMenu({ field, fields, conditions, onChange }: { - field?: Field; fields: Field[]; conditions: FilterTree; onChange: (conditions: FilterTree) => void; -}) { - const count = field ? activityFieldConditionCount(conditions.nodes, field.key) : countConditions(conditions.nodes); - const label = field?.label ?? "Filter"; - const focused = useMemo(() => { - if (!field || count) return conditions; - // An unfinished shared rule stays inactive until the builder commits a value. - return { ...conditions, nodes: [...conditions.nodes, - createFilterRule({ colId: field.key, op: opsForType(field.type)[0], value: "" })] }; - }, [conditions, field, count]); - const choices = field ? [field, ...fields.filter(item => item.key !== field.key)] : fields; - return ; -} - -function ActivityFilterToolbar({ user, fields, filters, resultsId, hasFilters, loading, refreshedAt, onChange, onClear, onRefresh }: { - user: SessionUser; fields: Field[]; filters: ActivityFilters; resultsId: string; hasFilters: boolean; - loading: boolean; refreshedAt: number | null; onChange: (patch: Partial) => void; - onClear: () => void; onRefresh: () => void; -}) { - const conditions: FilterTree = filters.conditions ?? { conj: "and", nodes: [] }; - const changeConditions = (conditions: FilterTree) => onChange({ conditions }); - return
- - {fields.filter(field => field.filterable !== false).map(field => )} - {!isAdmin(user) ? - User: {user.username} : null} - - - -
; -} - function ActivityPane({ user }: { user: SessionUser }) { const admin = isAdmin(user); const [users, setUsers] = useState([user.username]); @@ -261,7 +218,6 @@ function ActivityPane({ user }: { user: SessionUser }) { resultsRef.current?.closest(".set-body")?.scrollTo({ top: 0, behavior: "instant" }); }, [currentPage, filters]); function updateFilters(patch: Partial) { setFilters(value => ({ ...value, ...patch })); setPage(0); } - function clearFilters() { setFilters(DEFAULT_FILTERS); setPage(0); } function changePage(next: number) { setPage(next); resultsRef.current?.focus({ preventScroll: true }); @@ -269,16 +225,30 @@ function ActivityPane({ user }: { user: SessionUser }) { return

Activity log

- setRevision(value => value + 1)} /> +
+ + + {!admin ? + User: {user.username} : null} + + +
{denied ?

{problem || "You do not have access to this activity."}

: problem ?
{problem}
: loading ? : !filtered.length ?

{hasFilters ? "No activity matches your filters." : "No activity yet."}

- {hasFilters ? : null}
+ {hasFilters ? : null}
: <>
{groups.map(group =>

diff --git a/web/src/settings/SettingsModal.tsx b/web/src/settings/SettingsModal.tsx index cc53ae2155794431bd78af926d2feb87d2b31473..3f0afde1dec87af0686da8b2c1ff25b36b70b93d 100644 --- a/web/src/settings/SettingsModal.tsx +++ b/web/src/settings/SettingsModal.tsx @@ -61,7 +61,6 @@ import NotifyDestinations from "./NotifyDestinations"; import SecurityPane from "./SecurityPane"; import ActivityLogPage from "./ActivityLogPage"; import UsagePane from "./UsagePane"; -import BrandPane from "./BrandPane"; import OperatorPage from "../operator/OperatorPage"; /* ⛔ `StatementsPane` LEFT WITH ITS RAIL ROW (wave 35 · T38): the file is DELETED, not merely unmounted. An orphaned component with no importer is the shape this repo books as @@ -149,13 +148,6 @@ const IconUsage = ( ); -const IconBrand = ( - - - - - -); /** Sheet shortcuts — a small keyboard: a row of keys above a spacebar, not the padlock * "Your access" already wears (two authority glyphs in one rail would drift together). */ const IconShortcuts = ( @@ -924,7 +916,6 @@ export function SettingsModal({ ...(admin ? ([ ["usage", "Usage", IconUsage], - ["brand", "Brand", IconBrand], ["users", "Manage users", IconUsers], // R31 places field controls on each database, in Connectors or Data. // ⭐ WAVE 33, owner item 10 — placed by the owner, verbatim: *"under a @@ -1025,7 +1016,6 @@ export function SettingsModal({ {active === "security" ? : null} {active === "activity" ? : null} {active === "usage" ? : null} - {active === "brand" ? : null} {/* Wave 19 contract C2 — the wiring record, verified at the MOUNT SITE (the wave-14 lesson: an occurrence-grep proves an import, not a render): `settings/AdminPane.tsx → SettingsModal.tsx pane switch → diff --git a/web/src/settings/activityLog.css b/web/src/settings/activityLog.css index bdea1fca4d164b017a63e38f4b20bd780ca85d9f..c33f6391beee882f4156cffebe9515926ab87751 100644 --- a/web/src/settings/activityLog.css +++ b/web/src/settings/activityLog.css @@ -29,7 +29,6 @@ .activity-search input::placeholder { color: var(--lp-muted); } .activity-search:focus-within { outline: 2px solid var(--lp-blue-deep); outline-offset: 2px; } .activity-toolbar .cg-tb-btn { min-height: var(--set-control-height); } -.activity-toolbar .cg-tb-label { display: inline; } .activity-access-limit { display: inline-flex; align-items: center; diff --git a/web/src/settings/activityLog.ts b/web/src/settings/activityLog.ts index 39e929082e341d4723469f408992dc5fac5a5e2f..58dbd0fd07da399f1f66a58fb0dbdc11b4cc917a 100644 --- a/web/src/settings/activityLog.ts +++ b/web/src/settings/activityLog.ts @@ -1,6 +1,5 @@ import { API_V1, CREDENTIALS, UNAUTHORIZED_EVENT } from "../apiContract"; -import type { Field, FilterNode, FilterTree } from "../customer-grid/types"; -import { isFilterGroup } from "../customer-grid/types"; +import type { Field, FilterTree } from "../customer-grid/types"; import { matchFilterTree } from "../customer-grid/useVisibleRows"; export interface ActivityTarget { @@ -82,11 +81,6 @@ export function activityFilterFields(users: string[], admin: boolean): Field[] { ]; } -export function activityFieldConditionCount(nodes: FilterNode[], key: string): number { - return nodes.reduce((count, node) => count + (isFilterGroup(node) - ? activityFieldConditionCount(node.children, key) : Number(node.colId === key)), 0); -} - function localDay(ts: number): string { const date = new Date(ts); return `${date.getFullYear()}-${String(date.getMonth() + 1).padStart(2, "0")}-${String(date.getDate()).padStart(2, "0")}`; diff --git a/web/src/settings/permsModel.ts b/web/src/settings/permsModel.ts index 872a49c6e1de8989d1b1fe6d7eea6e2dcbaceeb3..bc73bf4ce204e59030cf635275cfb4cbb2b6fc32 100644 --- a/web/src/settings/permsModel.ts +++ b/web/src/settings/permsModel.ts @@ -56,7 +56,6 @@ export type SettingsSection = | "notifications" | "activity" | "usage" - | "brand" | "external" | "users" /** @@ -154,7 +153,7 @@ export function reachableSection( // `adminOnly` below would read `false` for it and hand back a section nothing renders. if (RETIRED_SECTIONS.includes(String(section)) || retiredSettingsRoute(section)) return "account"; // Connector credentials have moved to Setup; the server still enforces every write. - const adminOnly = section === "users" || section === "usage" || section === "brand"; + const adminOnly = section === "users" || section === "usage"; // ⚠ The cast is safe BECAUSE of the retired-section guard at the top: every value reaching here // is either a current member of the union or has already been degraded to "account". return adminOnly && !admin ? "account" : (section as SettingsSection); diff --git a/web/src/sheet/SheetGrid.tsx b/web/src/sheet/SheetGrid.tsx index 51d811ec8e1f203cfc18abe6d59698c86741c73b..d13f3bc390fa505d98cb4dd446a7b906a20ab57a 100644 --- a/web/src/sheet/SheetGrid.tsx +++ b/web/src/sheet/SheetGrid.tsx @@ -1494,8 +1494,7 @@ export default function SheetGrid({ // ⛔ THE FIND BOX'S CLOSE, THROUGH THE ONE ARBITER. Declared here, beside the state it closes, // and unconditionally, so the hook order never changes between renders. useEscapeClose(find !== null, () => setFind(null)); - // Undefined awaits selection bounds; null preserves an explicit dismissal. - const [anchor, setAnchor] = useState(undefined); + const [anchor, setAnchor] = useState(null); const [menu, setMenu] = useState(null); // S68-T28: clipboard knowledge belongs to this opening, never to the previous menu. const [menuClipboard, setMenuClipboard] = useState<{ owner: SheetMenu; empty: boolean } | null>(null); @@ -2144,16 +2143,9 @@ export default function SheetGrid({ }; }, [selection, cells, visCols, allVisRows, bandRows, getCellContent]); - const selected = useMemo(() => { - const cur = selection?.current?.cell; - if (!cur) return null; - const addr = addrAt(cur[0], cur[1]); - return { addr, cell: cellAt(tab, addr) }; - }, [addrAt, selection, tab]); - - const hasCellAnchor = anchor != null; + const hasCellAnchor = anchor !== null; useEffect(() => { - if ((!hasCellAnchor && (anchor !== undefined || selected?.cell?.kind !== "bound")) || !wrapRef.current) return; + if (!hasCellAnchor || !wrapRef.current) return; let frame = 0; const project = () => { cancelAnimationFrame(frame); @@ -2165,17 +2157,25 @@ export default function SheetGrid({ ? bandGridRef.current?.getBounds(item[0], item[1] + bandRows) : gridRef.current?.getBounds(item[0], item[1]); if (!bounds || ![bounds.x, bounds.y, bounds.width, bounds.height].every(Number.isFinite)) return; - setAnchor(prior => prior === null ? null : prior && (prior.x === bounds.x && prior.y === bounds.y + setAnchor(prior => !prior || (prior.x === bounds.x && prior.y === bounds.y && prior.width === bounds.width && prior.height === bounds.height) ? prior : bounds); }); }); }; - // Bound selections need their first anchor as well as layout updates, including the frozen grid. + // A cell click can collapse the navigation rail after Glide emitted its old viewport rect. + // Reproject the selected cell when that layout settles, including the frozen grid. const observer = new ResizeObserver(project); observer.observe(wrapRef.current); project(); return () => { observer.disconnect(); cancelAnimationFrame(frame); }; - }, [hasCellAnchor, anchor === undefined, selection, selected?.cell?.kind, bandRows, zoom]); + }, [hasCellAnchor, bandRows, zoom]); + + const selected = useMemo(() => { + const cur = selection?.current?.cell; + if (!cur) return null; + const addr = addrAt(cur[0], cur[1]); + return { addr, cell: cellAt(tab, addr) }; + }, [addrAt, selection, tab]); /** * The values needed by the selection door move as the workbook tree moves, but the door itself @@ -2295,8 +2295,7 @@ export default function SheetGrid({ // R11: the bar is anchored to the selection and unmounts when the selection changes. Clearing // the anchor here is what makes "disappears when the selection changes" true rather than // approximately true. - const sameCell = prior && accepted.current?.cell[0] === prior[0] && accepted.current.cell[1] === prior[1]; - setAnchor(sameCell ? null : undefined); + setAnchor(null); // ⭐ W52-T07: the paste highlight names what just happened, not what is still selected, so a // click elsewhere retires it. The copy marquee is the OTHER of the two and deliberately does // not clear here — see its own doc comment beside `copyMarquee`. diff --git a/web/src/sheet/SheetPage.tsx b/web/src/sheet/SheetPage.tsx index cf7f1a193d0cf3a90631150ce6cb8b6f9c38f0fa..23a2f1bc40d176667fcaad0634bf51dee59a0360 100644 --- a/web/src/sheet/SheetPage.tsx +++ b/web/src/sheet/SheetPage.tsx @@ -35,7 +35,6 @@ import { useCallback, useEffect, useId, useMemo, useRef, useState } from "react"; import type { PointerEvent as ReactPointerEvent, ReactNode } from "react"; import { TOAST_EVENT } from "../apiContract"; -import { viewAsActive } from "../settings/viewAs"; import "../styles/wave49/sheet.css"; import "../styles/wave50/sheet.css"; import "../styles/wave55/sheet.css"; @@ -2050,7 +2049,7 @@ export default function SheetPage({ address: mountAddress, embedded = false, onN // The index carries the server's effective role, including shares and public view access. const outlineRole = (index.workbooks.concat(index.sharedWithMe ?? []) .find(row => row.id === book?.id) as { role?: string } | undefined)?.role; - const canEditOutline = !viewAsActive() && (outlineRole === "owner" || outlineRole === "editor"); + const canEditOutline = outlineRole === "owner" || outlineRole === "editor"; const onRowGroups = useCallback((groups: SheetRowGroup[]) => { const base = bookRef.current; if (!base || !canEditOutline) return; diff --git a/web/src/shell/ShareDialog.tsx b/web/src/shell/ShareDialog.tsx index 41c290da318de17fc26e8ecfbea00a17071a9b46..e5edffc345cae3f85409d2a8e3213ce8d6b40528 100644 --- a/web/src/shell/ShareDialog.tsx +++ b/web/src/shell/ShareDialog.tsx @@ -42,7 +42,6 @@ const KIND_WORD: Record = { database: "database", field: "field", workbook: "workbook", - report: "report", }; // A newly-created field is posted through the standalone event queue. The @@ -86,10 +85,6 @@ const ROLE_BLURB: Record> = { view: "View only: can open this workbook and read every sheet in it. Cannot change a cell, a formula or a binding.", edit: "Can change this workbook's cells, formulas and bindings.", }, - report: { - view: "Can download this report.", - edit: "Reports cannot be edited.", - }, }; /** @@ -333,7 +328,7 @@ export default function ShareDialog({ ); const mayAdminister = !!state?.mayAdminister; - const grantableRoles = SHARE_ROLES.filter((role) => mayAdminister && (kind !== "report" || role === "view") && ( + const grantableRoles = SHARE_ROLES.filter((role) => mayAdminister && ( maxGrantableRole === "edit" || (maxGrantableRole === "view" && role === "view") )); const assign = useCallback(async (user: string) => { @@ -350,11 +345,9 @@ export default function ShareDialog({ }, [path, onToast]); const options = state ? addablePeople(state.people, entries) : []; const pick = state - ? kind === "report" - ? resolveShareRecipient(state.people, entries, recipientQuery, kind) - : resolveShareRecipient(state.people, entries, recipientQuery) + ? resolveShareRecipient(state.people, entries, recipientQuery) : null; - const canAddEveryone = kind !== "report" && !entries.some((entry) => entry.user === EVERYONE); + const canAddEveryone = !entries.some((entry) => entry.user === EVERYONE); const nameOf = (user: string) => user === "*" ? "Everyone" @@ -389,15 +382,13 @@ export default function ShareDialog({ spot already recorded twenty lines below. REWRITTEN, not stripped: the question the dialog answers is now the sentence's subject, which needs no dash at all. */}

- {kind !== "report" ? <>Who can reach {label}, and what they can do with it.{" "} : null} + Who can reach {label}, and what they can do with it.{" "} {kind === "workbook" ? WORKBOOK_BOUND_CELLS : kind === "database" ? "Sharing a database shares all of it: every record and every column. There is no per-row or per-column limit on a database grant." : kind === "field" ? "A shared column starts out hidden from everybody else on this database. This is what lets the people you choose see it and its values." - : kind === "report" - ? "Only the people you add here see this report." : "Sharing never widens past this workspace: everyone here can already open the surface it lives on."}

{kind === "view" ? ( @@ -489,13 +480,13 @@ export default function ShareDialog({ ))} : ( // Existing grants above the ceiling stay unchanged in every replacing PUT. - {e.role === "edit" ? "Can edit" : kind === "report" ? "Can download" : kind === "workbook" ? "View only" : "Can view"} + {e.role === "edit" ? "Can edit" : kind === "workbook" ? "View only" : "Can view"} )}
); } - -// Refusals reach the shared stack as error toasts. The callback is stable, so effects may depend on it. -export function useRefusalToast() { - const [messages, setMessages] = useState<{ id: number; text: string }[]>([]); - const sequence = useRef(0); - const delivered = useRef(0); - useEffect(() => { - if (!messages.length) return; - const through = messages[messages.length - 1].id; - for (const message of messages) { - if (message.id <= delivered.current) continue; - delivered.current = message.id; - const text = message.text; - window.dispatchEvent(new CustomEvent(TOAST_EVENT, { detail: { text, tone: "error" } })); - } - setMessages((current) => current.filter((message) => message.id > through)); - }, [messages]); - return useCallback((text: string) => { - if (!text) return; - const message = { id: ++sequence.current, text }; - setMessages((current) => [...current, message]); - }, []); -} diff --git a/web/src/shell/shareModel.ts b/web/src/shell/shareModel.ts index 95c9e9601f33683bf5bcbc696d7561d27fd14434..f682b2811394a731c5b6e44bae3fbf8cc2a41928 100644 --- a/web/src/shell/shareModel.ts +++ b/web/src/shell/shareModel.ts @@ -43,7 +43,7 @@ * `core.shares.KINDS` to match, and until it does the dialog says in a sentence that a workbook * cannot be shared yet rather than printing a status number. */ -export const SHARE_KINDS = ["view", "folder", "database", "field", "workbook", "report"] as const; +export const SHARE_KINDS = ["view", "folder", "database", "field", "workbook"] as const; export type ShareKind = (typeof SHARE_KINDS)[number]; /** The two roles, the same two words the view rail already speaks (R10). */ @@ -190,15 +190,13 @@ export function resolveShareRecipient( people: SharePerson[], entries: ShareEntry[], query: string, - kind: ShareKind = "view", ): string | null { const wanted = str(query).toLowerCase(); if (!wanted) return null; - if (kind === "report" && (wanted === EVERYONE || wanted === EVERYONE_LABEL.toLowerCase())) return null; const taken = new Set(entries.map((entry) => entry.user)); if (wanted === EVERYONE_LABEL.toLowerCase() && !taken.has(EVERYONE)) return EVERYONE; const matches = people.filter((person) => { - if (taken.has(person.user) || (kind === "report" && person.user === EVERYONE)) return false; + if (taken.has(person.user)) return false; const combined = `${person.name} (${person.user})`.toLowerCase(); return wanted === person.user || wanted === person.name.toLowerCase() || wanted === combined; }); diff --git a/web/src/styles/sdlc74/settings.css b/web/src/styles/sdlc74/settings.css index 5fc8a978e52481c09b01a57c52b9342ecbfebc26..f3ffea3394122b6d7fa0b79cb841d73a47a647da 100644 --- a/web/src/styles/sdlc74/settings.css +++ b/web/src/styles/sdlc74/settings.css @@ -66,15 +66,6 @@ font-size: var(--lp-fs-2xs); } -.set-brand-logo { - display: block; - max-width: 100%; - width: auto; - height: calc(var(--set-control-height) * 4); - object-fit: contain; - object-position: left center; -} - .set-actions { display: flex; flex-wrap: wrap; diff --git a/web/src/styles/sdlc82/amazon-projects.css b/web/src/styles/sdlc82/amazon-projects.css index a1268391dcb4bd2929fbd5e61028843ec1513c2a..63bb017f015cc188c71b3dc67329d8531802a38f 100644 --- a/web/src/styles/sdlc82/amazon-projects.css +++ b/web/src/styles/sdlc82/amazon-projects.css @@ -230,7 +230,8 @@ .lp-proj-title-input:disabled, .lp-proj-chip input:disabled, .lp-proj-chip select:disabled, -.lp-proj-notes:disabled { +.lp-proj-notes:disabled, +.lp-proj-percent input:disabled { opacity: 0.55; cursor: not-allowed; } @@ -569,6 +570,13 @@ border-bottom: 1px solid var(--lp-line); } +.lp-proj-target-line { + display: flex; + align-items: center; + gap: 10px; + min-width: 0; +} + .lp-proj-thumb { flex: none; width: 32px; diff --git a/web/src/styles/sdlc85/project-overview.css b/web/src/styles/sdlc85/project-overview.css index 06010a11a331ef08966c4d4d45541435ae832383..4cd73cd0b8a127ebd577ba25432b82524bcb7dd7 100644 --- a/web/src/styles/sdlc85/project-overview.css +++ b/web/src/styles/sdlc85/project-overview.css @@ -398,7 +398,10 @@ .lp-proj-setup-inline > svg { flex: none; } .lp-proj-setup-inline > select { width: auto; min-width: 0; } -.lp-proj-setup-input { +.lp-proj-setup-assignee { max-width: 320px; } + +.lp-proj-setup-input, +.lp-proj-setup-assignee > select { width: 100%; max-width: 100%; min-height: 32px; @@ -411,7 +414,8 @@ box-sizing: border-box; } -.lp-proj-setup-input:focus { +.lp-proj-setup-input:focus, +.lp-proj-setup-assignee > select:focus { border-color: var(--lp-line-strong); background: var(--lp-surface); } @@ -433,14 +437,17 @@ .lp-proj-title-input:focus-visible, .lp-proj-setup-input:focus-visible, -.lp-proj-setup-edit:focus-visible { +.lp-proj-setup-edit:focus-visible, +.lp-proj-setup-assignee > select:focus-visible { outline: 2px solid var(--lp-blue-deep); outline-offset: 2px; } .lp-proj-setup-input.lp-proj-setup-percent { width: 80px; font-variant-numeric: tabular-nums; } .lp-proj-setup-description { display: block; resize: vertical; min-height: 96px; } +.lp-proj-setup-assignee[data-empty] > select, .lp-proj-setup-input[data-empty] { color: var(--lp-muted); } +.lp-proj-setup-assignee option, .lp-proj-setup-input option { color: var(--lp-ink); } .lp-proj-setup-error { margin: 4px 0 8px; } @@ -472,7 +479,8 @@ .lp-proj-setup-edit[data-empty] { color: var(--lp-muted); } @media (hover: hover) and (pointer: fine) { - .lp-proj-setup-input:hover:not(:disabled) { border-color: var(--lp-line); } + .lp-proj-setup-input:hover:not(:disabled), + .lp-proj-setup-assignee > select:hover:not(:disabled) { border-color: var(--lp-line); } .lp-proj-setup-edit:hover:not(:disabled) { background: var(--lp-wash); } } @@ -744,6 +752,61 @@ min-width: 0; } +.lp-proj-card { + display: flex; + flex-direction: column; + gap: 10px; + padding: 16px 0; + border-top: 1px solid var(--lp-line); +} + +.lp-proj-progress-figure { + display: flex; + align-items: baseline; + justify-content: space-between; + gap: 8px; +} + +.lp-proj-percent { + display: inline-flex; + align-items: center; + gap: 4px; +} + +.lp-proj-percent input { + width: 64px; + min-height: 30px; + padding: 2px 6px; + border: 1px solid var(--lp-line); + border-radius: var(--lp-r-md); + background: var(--lp-surface); + color: var(--lp-ink); + font: inherit; + font-size: var(--lp-fs-md); + font-weight: 600; + font-variant-numeric: tabular-nums; +} + +.lp-proj-percent-text { + font-size: var(--lp-fs-md); + font-weight: 600; + font-variant-numeric: tabular-nums; +} + +.lp-proj-bar { + height: 6px; + border-radius: var(--lp-r-pill); + background: var(--lp-surface-2); + overflow: hidden; +} + +.lp-proj-bar > span { + display: block; + height: 100%; + border-radius: inherit; + background: var(--lp-proj-status-ink, var(--lp-blue-deep)); +} + .lp-proj-delete, .lp-proj-save { display: flex;