runloopable / platform /core /connector_sync.py
fsanyoto's picture
Deploy AIOS web (React glide grid + FastAPI slice)
463918b verified
Raw History Blame Contribute Delete
57.7 kB
"""Tenant-and-connection scoped connector sync lifecycle state. This data-layer module records what a provider runner did. It deliberately does not import or call providers: the API layer owns runners while every connector shares this durable vocabulary. """ from __future__ import annotations import datetime as _dt import hashlib import json import logging import threading import time import uuid from copy import deepcopy import core.store as store STATE_KEY = "connector_sync_state" STATES = ("queued", "syncing", "materializing", "ready", "partial", "error") _ERROR_LIMIT = 50 _DEFAULT_CONNECTION = "default" _PROVIDERS = {} # Ordinary requests coalesce per provider/tenant. Explicit events retain separate durable receipts. _SYNC_REQUESTS = {} _SYNC_REQUEST_RUNTIMES = {} _SYNC_NOT_BEFORE = {} _SYNC_REQUEST_LOCK = threading.Lock() _SYNC_WAKE = threading.Event() _SYNC_DUE_TIMER = None SYNC_REQUESTS_KEY = "connector_sync_requests_v1" NOTIFICATION_TARGETING_CONTRACT = "amazon-table-events-v1" # Ordinary requests the web process hands to the sync worker live in this member of the desk. REMOTE_MEMBER = "ordinary" REMOTE_KEEP_SECONDS = 3600 _REQUEST_LISTENERS = [] _REMOTE_IDS = {} _EVENT_DEDUPE_SECONDS = 600 def _target_tables(provider, tenant, rt, tables, event_id): from core.connector_spec import AMAZON_TABLES known = {row.source.removeprefix("amzn_") for row in AMAZON_TABLES if not row.native.startswith("derived:")} if (provider != "amazon_sp" or rt is None or rt.key != tenant or not isinstance(tables, list) or not tables or any(not isinstance(table, str) or table not in known for table in tables)): raise ValueError("amazon_notification_target_invalid") if not isinstance(event_id, str) or not event_id.strip() or len(tables) != 1: raise ValueError("Amazon notifications require one table and an event ID.") return list(tables) def _sync_request_document(current): document = deepcopy(current) if current is not None else {} if not isinstance(document, dict): raise ValueError("The stored sync request desk is unavailable.") for member in ("requests", "events", REMOTE_MEMBER): document.setdefault(member, {}) if not isinstance(document[member], dict): raise ValueError("The stored sync request desk is unavailable.") return document def _cache_sync_requests(rt, document): """Refresh this tenant's durable receipts under the desk lock without touching ordinary work.""" rows = {} for identity, row in document["requests"].items(): if not isinstance(row, dict) or row.get("id") != identity or row.get("tenant") != rt.key: raise ValueError("The stored sync receipt does not match its tenant or identity.") _target_tables(row.get("provider"), rt.key, rt, row.get("tables"), row.get("eventId")) rows[(row["provider"], rt.key, identity)] = deepcopy(row) for key in list(_SYNC_REQUESTS): if len(key) == 3 and key[1] == rt.key: del _SYNC_REQUESTS[key] _SYNC_REQUESTS.update(rows) _SYNC_REQUEST_RUNTIMES[rt.key] = rt def _wake_sync_requests(stamp, held=()): _arm_sync_wake(stamp) if any(row["state"] == "queued" and (row.get("notBefore") or 0) <= stamp and row.get("requestedBy") not in held for row in _SYNC_REQUESTS.values()): _SYNC_WAKE.set() def resume_held_requests(now=None): """Wake the resident loop for queued work a hold left waiting.""" with _SYNC_REQUEST_LOCK: _wake_sync_requests(time.time() if now is None else now) def restore_sync_requests(rt, *, now=None, requeue=True): """Restore durable events and requeue claims abandoned by a previous process. requeue=False only reads: a process that never claims must not requeue another one's claim. """ stamp = time.time() if now is None else now with _SYNC_REQUEST_LOCK: document = _sync_request_document(rt.get(SYNC_REQUESTS_KEY, fresh=True)) abandoned = {identity for identity, row in document["requests"].items() if requeue and row.get("state") == "running" and _SYNC_REQUESTS.get((row.get("provider"), rt.key, identity), {}).get("state") != "running"} if abandoned: def restore(current): doc = _sync_request_document(current) for identity in abandoned: row = doc["requests"].get(identity) if row and row["state"] == "running": row.update(state="queued", startedAt=None, claimToken="", restoredAt=stamp) return doc document = rt.update(SYNC_REQUESTS_KEY, restore, flush="sync") _cache_sync_requests(rt, document) _wake_sync_requests(stamp) return [deepcopy(row) for row in document["requests"].values()] def _arm_sync_wake(now): """Called under the desk lock. A future claim wakes the resident loop once, at its floor. Bootstrap re-arms this disposable alarm from the provider's durable not-before marker. The alarm only wakes the single writer; it neither queues work nor calls a provider. """ global _SYNC_DUE_TIMER if _SYNC_DUE_TIMER is not None: _SYNC_DUE_TIMER.cancel() _SYNC_DUE_TIMER = None due = [row["notBefore"] for row in _SYNC_REQUESTS.values() if row["state"] == "queued" and row.get("notBefore") and row["notBefore"] > now] if due: _SYNC_DUE_TIMER = threading.Timer(min(due) - now, _SYNC_WAKE.set) _SYNC_DUE_TIMER.daemon = True _SYNC_DUE_TIMER.start() def defer_sync(provider, tenant, not_before): """Install a provider's durable checkpoint boundary, including on process bootstrap. The provider owns persistence and restores this boundary before re-queuing its work. This module only claims intents; it never reads a provider mirror. """ with _SYNC_REQUEST_LOCK: key = (provider, tenant) if not_before is None: _SYNC_NOT_BEFORE.pop(key, None) else: _SYNC_NOT_BEFORE[key] = float(not_before) receipt = _SYNC_REQUESTS.get(key) if receipt and receipt["state"] == "queued": receipt["notBefore"] = _SYNC_NOT_BEFORE.get(key) _arm_sync_wake(time.time()) if not not_before or float(not_before) <= time.time(): _SYNC_WAKE.set() def sync_not_before(provider, tenant): """The provider's installed not-before floor as an epoch, or None.""" with _SYNC_REQUEST_LOCK: return _SYNC_NOT_BEFORE.get((provider, tenant)) def _clear_sync_defer(provider, tenant): """Remove one obsolete provider hold under the desk lock, retaining explicit continuation waits.""" key = (provider, tenant) held = _SYNC_NOT_BEFORE.pop(key, None) receipt = _SYNC_REQUESTS.get(key) if (held is not None and receipt and receipt["state"] == "queued" and receipt.get("requestedBy") != "first load" and receipt.get("notBefore") == held): receipt["notBefore"] = None def clear_sync_defer(provider, tenant): """Release one provider hold after its producer adopts partition-specific waits.""" with _SYNC_REQUEST_LOCK: _clear_sync_defer(provider, tenant) _wake_sync_requests(time.time()) def register_request_listener(fn): """Call fn(receipt, connection_id) after each accepted ordinary request, outside the desk lock.""" if fn not in _REQUEST_LISTENERS: _REQUEST_LISTENERS.append(fn) def request_sync(provider, tenant, *, requested_by="", now=None, not_before=None, rt=None, tables=None, event_id="", connection_id=None, wait_cause=""): """Queue ordinary work or durably acknowledge one tenant-bound Amazon event. `wait_cause` is the safe sentence a receipt held by its not-before floor shows instead of the bare stamp. """ receipt = _request_sync(provider, tenant, requested_by=requested_by, now=now, not_before=not_before, rt=rt, tables=tables, event_id=event_id, wait_cause=wait_cause) if tables is None and receipt.get("accepted"): for listener in list(_REQUEST_LISTENERS): listener(dict(receipt), connection_id) return receipt def _request_sync(provider, tenant, *, requested_by="", now=None, not_before=None, rt=None, tables=None, event_id="", wait_cause=""): if provider not in _PROVIDERS: raise ValueError("This connector is not registered for sync.") tenant = str(tenant or "").strip() if not tenant: raise ValueError("A tenant is required for sync.") if tables is None and event_id: raise ValueError("Amazon notifications require an explicit table target.") if tables is not None: targets = _target_tables(provider, tenant, rt, tables, event_id) stamp, identity = time.time() if now is None else now, uuid.uuid4().hex event_key = hashlib.sha256(json.dumps([provider, event_id], separators=(",", ":")).encode()).hexdigest() receipt = {"id": identity, "provider": provider, "tenant": tenant, "tables": targets, "eventId": event_id, "requestedBy": str(requested_by or ""), "requestedAt": stamp, "notBefore": float(not_before) if not_before is not None else None, "state": "queued", "startedAt": None, "finishedAt": None, "detail": "", "accepted": False, "claimToken": ""} def enqueue_event(current): document = _sync_request_document(current) event = document["events"].get(event_key) if event and stamp < event["firstAcceptedAt"] + _EVENT_DEDUPE_SECONDS: if event["requestId"] not in document["requests"]: raise ValueError("The stored Amazon event receipt is unavailable.") return document document["requests"][identity] = deepcopy(receipt) document["events"][event_key] = {"requestId": identity, "firstAcceptedAt": stamp} return document with _SYNC_REQUEST_LOCK: document = rt.update(SYNC_REQUESTS_KEY, enqueue_event, flush="sync") _cache_sync_requests(rt, document) accepted_id = document["events"][event_key]["requestId"] result = deepcopy(document["requests"][accepted_id]) _wake_sync_requests(stamp) return {**result, "accepted": accepted_id == identity, "duplicate": accepted_id != identity} with _SYNC_REQUEST_LOCK: if provider == "amazon_sp" and requested_by != "first load": # Amazon's history floor belongs to owed tables, never a loaded-table refresh. _clear_sync_defer(provider, tenant) previous = _SYNC_REQUESTS.get((provider, tenant)) replaces_amazon_history = (provider == "amazon_sp" and previous and previous.get("requestedBy") == "first load" and requested_by != "first load") if previous and previous["state"] in ("queued", "running"): if not (previous["state"] == "queued" and previous.get("requestedBy") == "first load" and requested_by != "first load"): return {**_request_wait(previous), "accepted": False} receipt = {"id": uuid.uuid4().hex, "provider": provider, "tenant": tenant, "state": "queued", "requestedAt": time.time() if now is None else now, "notBefore": max(float(not_before or 0), _SYNC_NOT_BEFORE.get((provider, tenant), 0), float(previous.get("notBefore") or 0) if previous and previous["state"] == "queued" and not replaces_amazon_history else 0) or None, "startedAt": None, "finishedAt": None, "detail": "", "requestedBy": str(requested_by or ""), "accepted": True} if requested_by == "first load" and previous and previous.get("firstLoad"): # A continuation keeps the safe numeric progress visible while waiting for the writer. receipt.update(detail=previous["detail"], firstLoad=dict(previous["firstLoad"])) if wait_cause: receipt.update(detail=str(wait_cause)[:300], waitCause=str(wait_cause)[:300]) _SYNC_REQUESTS[(provider, tenant)] = receipt stamp = time.time() if now is None else now _arm_sync_wake(stamp) if (receipt.get("notBefore") or 0) <= stamp: _SYNC_WAKE.set() return _request_wait(receipt, now=stamp) def _request_order(receipt): """Manual work precedes first-load continuations, with FIFO within each class.""" return receipt.get("requestedBy") == "first load", receipt.get("requestedAt") or 0 def _request_wait(receipt, *, now=None): """Describe a queued receipt's executing competitor or provider floor without claiming a start.""" result = deepcopy(receipt) result.setdefault("waitingFor", None) if receipt["state"] != "queued": result["waitingFor"] = None return result running = [row for row in _SYNC_REQUESTS.values() if row["tenant"] == receipt["tenant"] and row["state"] == "running" and row.get("startedAt") is not None] if running: active = max(running, key=lambda row: row["startedAt"]) started = active["startedAt"] identity = active.get("id") or f"{active['provider']}:{started}" result["waitingFor"] = {"id": identity, "provider": active["provider"], "startedAt": started} stamp = _dt.datetime.fromtimestamp(started, _dt.timezone.utc).isoformat() result["detail"] = f"Waiting for {active['provider']} receipt {identity}, started at {stamp}." elif result.get("waitingFor"): return result elif (receipt.get("notBefore") or 0) > (time.time() if now is None else now): stamp = _dt.datetime.fromtimestamp(receipt["notBefore"], _dt.timezone.utc).isoformat() result["detail"] = receipt.get("waitCause") or f"Waiting for the provider not-before floor at {stamp}." elif result.get("detail", "").startswith("Waiting for "): result["detail"] = "" return result def sync_request(provider, tenant): """Return a detached current receipt, or None.""" with _SYNC_REQUEST_LOCK: rows = [row for row in _SYNC_REQUESTS.values() if row["provider"] == provider and row["tenant"] == tenant] receipt = next((row for row in rows if row["state"] == "running"), None) if receipt is None: queued = [row for row in rows if row["state"] == "queued"] receipt = min(queued, key=_request_order) if queued else _SYNC_REQUESTS.get((provider, tenant)) if receipt is None: return None return {**_request_wait(receipt), "accepted": False} def take_sync_requests(*, now=None, limit=None, exclude_providers=(), refresh_remote=False, exclude_requested_by=()): """Claim manual work first, preserving provider floors, tenant claims and event siblings. `exclude_requested_by` leaves those receipts queued and does not wake the loop for them. """ if refresh_remote: with _SYNC_REQUEST_LOCK: runtimes = list(_SYNC_REQUEST_RUNTIMES.values()) for rt in runtimes: try: poll_remote_requests(rt, now=now) except Exception as exc: logging.getLogger(__name__).warning("Sync desk refresh unavailable for %s (%s)", rt.key, type(exc).__name__) with _SYNC_REQUEST_LOCK: _SYNC_WAKE.clear() claimed = [] stamp = time.time() if now is None else now busy = {(row["provider"], row["tenant"]) for row in _SYNC_REQUESTS.values() if row["state"] == "running"} for receipt in sorted(list(_SYNC_REQUESTS.values()), key=_request_order): if limit is not None and len(claimed) >= limit: break pair = (receipt["provider"], receipt["tenant"]) if (receipt["provider"] in exclude_providers or pair in busy or receipt["state"] != "queued" or receipt.get("requestedBy") in exclude_requested_by or max(receipt.get("notBefore") or 0, _SYNC_NOT_BEFORE.get(pair, 0)) > stamp): continue if receipt.get("tables") is not None: rt = _SYNC_REQUEST_RUNTIMES[receipt["tenant"]] identity, token = receipt["id"], uuid.uuid4().hex def claim_event(current): document = _sync_request_document(current) rows = [row for row in document["requests"].values() if (row["provider"], row["tenant"]) == pair] if any(row["state"] == "running" for row in rows): return document eligible = [row for row in rows if row["state"] == "queued" and (row.get("notBefore") or 0) <= stamp] row = min(eligible, key=_request_order) if eligible else None if row is not None and row["id"] == identity: row.update(state="running", startedAt=stamp, claimToken=token, accepted=False) return document document = rt.update(SYNC_REQUESTS_KEY, claim_event, flush="sync") _cache_sync_requests(rt, document) receipt = document["requests"].get(identity, {}) if receipt.get("claimToken") != token: continue else: receipt.update(state="running", startedAt=stamp, accepted=False) receipt["waitingFor"] = None if receipt.get("detail", "").startswith("Waiting for "): receipt["detail"] = "" claimed.append(deepcopy(receipt)) busy.add(pair) _wake_sync_requests(stamp, exclude_requested_by) return claimed def finish_sync_request(provider, tenant, *, ok, detail="", now=None, first_load=None, request_id=None): """Finish a claimed intent. The API runner supplies a safe cause sentence, never raw errors.""" with _SYNC_REQUEST_LOCK: receipt = next((row for row in _SYNC_REQUESTS.values() if row["provider"] == provider and row["tenant"] == tenant and row["state"] == "running" and (request_id is None or row.get("id") == request_id)), None) if receipt is None: return None stamp = time.time() if now is None else now completion = {"state": "done" if ok else "failed", "detail": str(detail or "")[:300], "finishedAt": stamp, "accepted": False} if first_load is not None: completion["firstLoad"] = dict(first_load) if receipt.get("tables") is not None: rt = _SYNC_REQUEST_RUNTIMES[tenant] identity, token = receipt["id"], receipt["claimToken"] def finish_event(current): document = _sync_request_document(current) row = document["requests"].get(identity) if row and row["state"] == "running" and row.get("claimToken") == token: row.update(completion) return document document = rt.update(SYNC_REQUESTS_KEY, finish_event, flush="sync") _cache_sync_requests(rt, document) receipt = document["requests"].get(identity, {}) if receipt.get("claimToken") != token or receipt.get("finishedAt") != stamp: return None else: receipt.update(completion) result = deepcopy(receipt) result["waitingFor"] = None _wake_sync_requests(stamp) connector = {"amazon_sp": "amazon", "meta_ads": "meta"}.get(provider) if connector: try: from core.sync_events import emit except ImportError: pass # C20: lane J supplies this module at combine. else: try: emit(tenant, connector, bool(ok), result["detail"], _dt.datetime.fromtimestamp(result["finishedAt"], _dt.timezone.utc).isoformat()) except Exception: logging.getLogger(__name__).exception("Sync completion listener failed") return result def sync_wake_event(): """The event the resident thread waits on between passes.""" return _SYNC_WAKE def queue_remote_request(rt, receipt, connection_id=None, *, now=None): """Web side: durably hand one accepted ordinary request to the sync worker process.""" stamp = time.time() if now is None else now identity = receipt.get("id") or uuid.uuid4().hex entry = {"id": identity, "provider": receipt["provider"], "tenant": rt.key, "requestedBy": str(receipt.get("requestedBy") or ""), "requestedAt": float(receipt.get("requestedAt") or stamp), "notBefore": receipt.get("notBefore"), "connectionId": str(connection_id) if connection_id else None, "state": "queued", "startedAt": None, "finishedAt": None, "detail": ""} def add(current): document = _sync_request_document(current) document[REMOTE_MEMBER][identity] = entry return document rt.update(SYNC_REQUESTS_KEY, add, flush="sync") with _SYNC_REQUEST_LOCK: _REMOTE_IDS[(receipt["provider"], rt.key)] = identity return identity def fail_remote_request(provider, tenant, detail, *, now=None): """Web side: a hand-off that could not be saved finishes its receipt instead of hanging.""" with _SYNC_REQUEST_LOCK: receipt = _SYNC_REQUESTS.get((provider, tenant)) if receipt and receipt["state"] == "queued": receipt.update(state="failed", detail=str(detail)[:300], finishedAt=time.time() if now is None else now) def _remote_progress(entry, receipt): """The worker receipt's progress for one entry, or None when that run does not cover it.""" started = receipt.get("startedAt") if receipt else None if (started is None or entry["requestedAt"] > started or receipt["state"] not in ("running", "done", "failed") or (entry.get("requestedBy") != "first load" and receipt.get("requestedBy") == "first load")): return None progress = {"state": receipt["state"], "startedAt": started, "finishedAt": receipt.get("finishedAt"), "detail": receipt.get("detail") or "", "waitingFor": None} if receipt.get("firstLoad"): progress["firstLoad"] = dict(receipt["firstLoad"]) return progress def poll_remote_requests(rt, *, now=None): """Worker side: admit queued hand-offs, publish their progress, prune old ones. An entry leaves the queue only when a run that started after it was requested finishes. Returns the entries admitted as new local receipts. """ stamp = time.time() if now is None else now admitted, changes, pruned = [], {}, [] with _SYNC_REQUEST_LOCK: document = _sync_request_document(rt.get(SYNC_REQUESTS_KEY, fresh=True)) for identity, entry in sorted(document[REMOTE_MEMBER].items(), key=lambda item: _request_order(item[1])): key = (entry.get("provider"), rt.key) if entry.get("tenant") != rt.key or key[0] not in _PROVIDERS: continue if entry["state"] in ("done", "failed"): if stamp - float(entry.get("finishedAt") or 0) > REMOTE_KEEP_SECONDS: pruned.append(identity) continue manual_amazon = key[0] == "amazon_sp" and entry.get("requestedBy") != "first load" if manual_amazon: _clear_sync_defer(*key) receipt = _SYNC_REQUESTS.get(key) # Promote under the same lock as claims, so a continuation cannot win a manual hand-off race. if (receipt and receipt["state"] == "queued" and receipt.get("requestedBy") == "first load" and entry.get("requestedBy") != "first load"): receipt.update(id=identity, requestedBy=entry.get("requestedBy") or "", requestedAt=entry["requestedAt"], connectionId=entry.get("connectionId"), notBefore=max(0 if manual_amazon else float(receipt.get("notBefore") or 0), float(entry.get("notBefore") or 0), _SYNC_NOT_BEFORE.get(key, 0)) or None) admitted.append(deepcopy(entry)) progress = _remote_progress(entry, receipt) if progress is not None: if any(entry.get(name) != value for name, value in progress.items()): changes[identity] = progress continue # A "running" entry with no local run was abandoned by a worker restart: admit it again. if (entry["state"] in ("queued", "running") and not (receipt and receipt["state"] in ("queued", "running"))): local = {"id": identity, "provider": key[0], "tenant": rt.key, "state": "queued", "requestedAt": entry["requestedAt"], "notBefore": max(float(entry.get("notBefore") or 0), _SYNC_NOT_BEFORE.get(key, 0)) or None, "startedAt": None, "finishedAt": None, "detail": "", "connectionId": entry.get("connectionId"), "requestedBy": entry.get("requestedBy") or "", "accepted": True} if receipt and receipt.get("firstLoad") and local["requestedBy"] == "first load": local.update(detail=receipt["detail"], firstLoad=dict(receipt["firstLoad"])) _SYNC_REQUESTS[key] = local _SYNC_REQUEST_RUNTIMES.setdefault(rt.key, rt) admitted.append(deepcopy(entry)) receipt = local waiting = _request_wait({**entry, "state": "queued", "startedAt": None, "finishedAt": None, "waitingFor": None, "notBefore": max(float(entry.get("notBefore") or 0), _SYNC_NOT_BEFORE.get(key, 0), float(receipt.get("notBefore") or 0) if receipt and receipt["state"] == "queued" else 0) or None}, now=stamp) fields = {name: waiting.get(name) for name in ("state", "startedAt", "finishedAt", "notBefore", "detail", "waitingFor")} if any(entry.get(name) != value for name, value in fields.items()): changes[identity] = fields if changes or pruned: def publish(current): doc = _sync_request_document(current) for identity, progress in changes.items(): if identity in doc[REMOTE_MEMBER]: doc[REMOTE_MEMBER][identity].update(progress) for identity in pruned: doc[REMOTE_MEMBER].pop(identity, None) return doc rt.update(SYNC_REQUESTS_KEY, publish, flush="sync") if admitted: _wake_sync_requests(stamp) return admitted def apply_remote_progress(rt): """Web side: copy the worker's published progress onto this process's receipts.""" document = _sync_request_document(rt.get(SYNC_REQUESTS_KEY, fresh=True)) applied = 0 with _SYNC_REQUEST_LOCK: for identity, entry in document[REMOTE_MEMBER].items(): key = (entry.get("provider"), rt.key) receipt = _SYNC_REQUESTS.get(key) if _REMOTE_IDS.get(key) != identity or receipt is None: continue fields = {name: entry.get(name) for name in ("state", "startedAt", "finishedAt", "detail", "notBefore", "waitingFor")} if entry.get("firstLoad"): fields["firstLoad"] = dict(entry["firstLoad"]) if any(receipt.get(k) != v for k, v in fields.items()): receipt.update(fields) applied += 1 # Event receipts the worker claimed or finished, refreshed without requeueing anything. _cache_sync_requests(rt, document) return applied # One receipt per immutable database identity. No credentials, session, or source values. JOBS_KEY = "connector_materialization_jobs" MAX_ATTEMPTS = 3 LEASE_SECONDS = 300 LIFECYCLE_STATES = ("configured", "catalogue-ready", "queued", "syncing", "ready", "failed") def job_key(provider, connection_id, database_key): return json.dumps([_key(provider), _slot(connection_id), _key(database_key)], separators=(",", ":")) def _job_public(row): answer = {k: v for k, v in row.items() if k != "leaseToken"} # Older status readers only know error. Keep one persisted cause and project it there. if row.get("state") == "queued" and row.get("stall"): answer["error"] = row["stall"]["cause"] answer["retryable"] = (row.get("state") == "failed" and int(row.get("attempts", 0)) < MAX_ATTEMPTS) answer["maxAttempts"] = MAX_ATTEMPTS if row.get("state") == "failed" and not answer["retryable"]: answer["error"] = "Sync attempts exhausted. Check the connection and update its settings." return answer def jobs(rt, provider=None, connection_id=None): """Persisted status, including receipts left by a previous process.""" document = rt.get(JOBS_KEY, fresh=True) or {} return [_job_public(row) for row in document.values() if isinstance(row, dict) and (provider is None or row.get("provider") == provider) and (connection_id is None or row.get("connectionId") == _slot(connection_id))] def job_summary(receipts): """Aggregate only durable facts; a retry of ready work must never claim it is queued.""" if not receipts: return None states = {row.get("state") for row in receipts} state = next((value for value in ("syncing", "queued", "failed", "ready", "catalogue-ready", "configured") if value in states), "configured") errors = [row["error"] for row in receipts if row.get("error")] stamp = max(row.get("at", 0) for row in receipts) return {"state": state, "at": _dt.datetime.fromtimestamp(stamp, _dt.timezone.utc).isoformat(), "retryable": any(row.get("retryable") for row in receipts), "databases": [row["databaseKey"] for row in receipts if row.get("state") == "ready"], "detail": f"Database sync {state}", "errors": errors, "errorsTotal": len(errors)} def configuration_revision(provider, connection_id, database_key, model, field_choices): """Fingerprint this database's field choices, never rows or sibling configuration.""" identity = database_identity(provider, connection_id, database_key, model) selected = [] for key, value in (field_choices or {}).items(): scoped = parse_field_key(key) if scoped and scoped[:4] == (identity["provider"], identity["connectionId"], identity["databaseKey"], identity["model"]): selected.append((scoped[4], value is True)) return hashlib.sha256(json.dumps(sorted(selected), separators=(",", ":")).encode()).hexdigest() def enqueue(rt, provider, connection_id, database_key, *, model="", retry=False, changed=False, revision=None, now=None): """A Save is idempotent. Failed work requires an explicit, bounded retry.""" identity = database_identity(provider, connection_id, database_key, model) key = job_key(provider, connection_id, database_key) stamp, receipt = time.time() if now is None else now, uuid.uuid4().hex def apply(current): doc = dict(current or {}) previous = doc.get(key) or {} updated = changed or (revision is not None and previous.get("revision") != revision) if not updated and previous.get("state") in ("queued", "syncing", "ready"): return doc attempts = int(previous.get("attempts", 0)) if not updated and previous.get("state") == "failed" and (not retry or attempts >= MAX_ATTEMPTS): return doc doc[key] = {**identity, "id": receipt, "state": "queued", "at": stamp, "attempts": attempts if retry and not updated else 0, "leaseUntil": 0, "revision": revision, "leaseToken": "", "error": "", "workspaceChanged": False} return doc landed = rt.update(JOBS_KEY, apply, flush="sync") return _job_public(landed[key]) def stall(rt, candidate, cause, *, now=None): """Record WHY queued work cannot start, without consuming an attempt.""" key = job_key(candidate["provider"], candidate["connectionId"], candidate["databaseKey"]) stamp = time.time() if now is None else now sentence = str(cause or "")[:300] def eligible(row): return (row.get("id") == candidate.get("id") and (row.get("state") == "queued" or row.get("state") == "syncing" and row.get("leaseUntil", 0) <= stamp)) previous = (rt.get(JOBS_KEY, fresh=True) or {}).get(key) or {} # Avoid even submitting a store update on the unchanged 15-second drain pass. if not eligible(previous) or (previous.get("stall") or {}).get("cause") == sentence: return _job_public(previous) def apply(current): doc = dict(current or {}) row = dict(doc.get(key) or {}) if not eligible(row) or (row.get("stall") or {}).get("cause") == sentence: return doc row.update(state="queued", leaseUntil=0, leaseToken="", stall={"at": stamp, "cause": sentence}) doc[key] = row return doc landed = rt.update(JOBS_KEY, apply, flush="sync") return _job_public(landed.get(key) or {}) def claim(rt, candidate, *, now=None): """Atomically claim queued or abandoned work; replay callbacks have no side effects.""" key = job_key(candidate["provider"], candidate["connectionId"], candidate["databaseKey"]) stamp, token = time.time() if now is None else now, uuid.uuid4().hex def apply(current): doc = dict(current or {}) row = dict(doc.get(key) or {}) if row.get("id") != candidate.get("id"): return doc if row.get("state") != "queued" and not ( row.get("state") == "syncing" and row.get("leaseUntil", 0) <= stamp): return doc row.pop("stall", None) if int(row.get("attempts", 0)) >= MAX_ATTEMPTS: row.update(state="failed", at=stamp, leaseToken="", leaseUntil=0, error="Sync attempts exhausted. Check the connection and update its settings.") else: row.update(state="syncing", at=stamp, attempts=int(row.get("attempts", 0)) + 1, leaseToken=token, leaseUntil=stamp + LEASE_SECONDS) doc[key] = row return doc landed = rt.update(JOBS_KEY, apply, flush="sync") row = landed.get(key) or {} return dict(row) if row.get("leaseToken") == token else None def owns(rt, job, *, now=None): row = (rt.get(JOBS_KEY, fresh=True) or {}).get(job_key( job["provider"], job["connectionId"], job["databaseKey"])) or {} return (row.get("id") == job.get("id") and row.get("state") == "syncing" and row.get("leaseToken") == job.get("leaseToken") and row.get("leaseUntil", 0) > (time.time() if now is None else now)) def progress(rt, job, *, rows, total): """Record published rows only while this worker still owns its unexpired lease.""" key = job_key(job["provider"], job["connectionId"], job["databaseKey"]) stamp = time.time() def apply(current): doc = dict(current or {}) row = dict(doc.get(key) or {}) if (row.get("id") != job.get("id") or row.get("state") != "syncing" or row.get("leaseToken") != job.get("leaseToken") or row.get("leaseUntil", 0) <= stamp): return doc row["progress"] = {"rows": rows, "total": total, "at": stamp} doc[key] = row return doc landed = rt.update(JOBS_KEY, apply, flush="sync") return _job_public(landed.get(key) or {}) def finish(rt, job, *, success=False, renew=False, interrupted=False, cause="", now=None): """A stale worker cannot acknowledge, renew, or overwrite a newer receipt.""" key = job_key(job["provider"], job["connectionId"], job["databaseKey"]) stamp = time.time() if now is None else now def apply(current): doc = dict(current or {}) row = dict(doc.get(key) or {}) if (row.get("id") != job.get("id") or row.get("state") != "syncing" or row.get("leaseToken") != job.get("leaseToken") or row.get("leaseUntil", 0) <= stamp): return doc if renew: row["leaseUntil"] = stamp + LEASE_SECONDS elif interrupted: # Disconnect suspends the current attempt, preserving earlier failures and identity. row.update(state="queued", at=stamp, attempts=max(0, int(row.get("attempts", 0)) - 1), leaseUntil=0, leaseToken="", workspaceChanged=False, error="") else: row.update(state="ready" if success else "failed", at=stamp, leaseUntil=0, leaseToken="", workspaceChanged=bool(success), error="" if success else (str(cause or "")[:300] or "Rows could not be synced. Check the connection and retry.")) doc[key] = row return doc landed = rt.update(JOBS_KEY, apply, flush="sync") return _job_public(landed.get(key) or {}) def cancel(rt, provider, connection_id, database_key): key = job_key(provider, connection_id, database_key) stamp = time.time() def apply(current): doc = dict(current or {}) if key in doc: doc[key] = {**doc[key], "state": "configured", "at": stamp, "leaseToken": "", "leaseUntil": 0, "workspaceChanged": False, "error": ""} doc[key].pop("stall", None) doc[key].pop("progress", None) return doc rt.update(JOBS_KEY, apply, flush="sync") def _st(st): return st if st is not None else store def _key(value): return str(value or "").strip() def _slot(connection_id): return _key(connection_id) or _DEFAULT_CONNECTION def _target(rt, st): return st if st is not None else rt def database_identity(provider, connection_id, database_key, model=""): """Source-neutral identity; labels and sync state never participate in a database key.""" if not _key(provider) or not _key(database_key): raise ValueError("provider and database key are required") return {"provider": _key(provider), "connectionId": _slot(connection_id), "databaseKey": _key(database_key), "model": _key(model)} def field_key(provider, connection_id, database_key, model, raw_name): """An opaque, collision-free key for one source field in one database.""" identity = database_identity(provider, connection_id, database_key, model) if not identity["model"] or not _key(raw_name): raise ValueError("model and raw field name are required") return json.dumps([identity["provider"], identity["connectionId"], identity["databaseKey"], identity["model"], _key(raw_name)], separators=(",", ":"), ensure_ascii=True) def parse_field_key(value): """Read a scoped field key, or return None for a legacy model.field key.""" try: parts = json.loads(value) except (TypeError, ValueError): return None if not isinstance(parts, list) or len(parts) != 5 or not all( isinstance(part, str) and part for part in parts): return None return (*parts,) def _public(provider, connection_id, row): if not isinstance(row, dict): return None answer = dict(row) answer["provider"] = provider answer["connectionId"] = connection_id # RULE: retryability follows the state. Storing it would let the two answers drift. answer["retryable"] = answer.get("state") in ("error", "partial") return answer def record(rt, provider, connection_id, state, *, detail="", errors=(), databases=(), st=None): """Record one lifecycle transition and return its defensive public representation.""" provider_key = _key(provider) if not provider_key: raise ValueError("provider is required") if state not in STATES: # RULE: unknown states are refused. A free-form status eventually stores a typo that the # UI cannot render as one of C6's six observable states. raise ValueError(f"unknown connector sync state: {state}") connection_key = _slot(connection_id) all_errors = [str(value) for value in (errors or ())] visible_errors = all_errors[:_ERROR_LIMIT] now = _dt.datetime.now(_dt.timezone.utc).isoformat() written = {} def _apply(current): document = dict(current or {}) providers_doc = dict(document.get(provider_key) or {}) previous = providers_doc.get(connection_key) previous_state = previous.get("state") if isinstance(previous, dict) else None row = { "state": state, # RULE: stamp every transition and name its predecessor. A resting state alone cannot # tell the UI whether work advanced, retried, or regressed. "at": now, "from": previous_state, "detail": str(detail or ""), # RULE: partial and error stay distinct. Partial exposes databases usable now; # error means no usable set was produced. "errors": visible_errors, # RULE: retain up to 50 errors and report the full count. A storage limit must be # observable rather than silently turning additional failures into zero information. "errorsTotal": len(all_errors), # These are governed table keys, not copied provider rows. In Odoo, row_limit == 0 # keeps 963,783 GL lines out of the editable user_tables document. "databases": [str(value) for value in (databases or ()) if str(value)], } providers_doc[connection_key] = row # RULE: key state by provider and connection. The measured legacy defect is that # odoo_sync_status is one tenant document, so a second Odoo connection overwrites the # first one's status. The default slot matches Odoo's existing catalog convention. document[provider_key] = providers_doc written.update(row) return document _st(_target(rt, st)).update(STATE_KEY, _apply, flush="sync") return _public(provider_key, connection_key, written) def read(rt, provider, connection_id=None, st=None): """Read one connection defensively; store failures mean no recorded state, never a 500.""" try: provider_key = _key(provider) connection_key = _slot(connection_id) document = _st(_target(rt, st)).get(STATE_KEY) or {} providers_doc = document.get(provider_key) if isinstance(document, dict) else None row = providers_doc.get(connection_key) if isinstance(providers_doc, dict) else None return _public(provider_key, connection_key, row) except Exception: # noqa: BLE001 - status reads cannot take down their caller return None def read_all(rt, st=None): """Read every provider and connection defensively.""" try: document = _st(_target(rt, st)).get(STATE_KEY) or {} answer = {} for provider_key, connections in document.items(): if not isinstance(connections, dict): continue public_connections = {} for connection_key, row in connections.items(): public = _public(str(provider_key), str(connection_key), row) if public is not None: public_connections[str(connection_key)] = public if public_connections: answer[str(provider_key)] = public_connections return answer except Exception: # noqa: BLE001 - status reads cannot take down their caller return {} def clear(rt, provider, connection_id, st=None): """Remove only the named provider/connection state.""" provider_key = _key(provider) connection_key = _slot(connection_id) def _drop(current): document = dict(current or {}) connections = dict(document.get(provider_key) or {}) connections.pop(connection_key, None) if connections: document[provider_key] = connections else: document.pop(provider_key, None) return document _st(_target(rt, st)).update(STATE_KEY, _drop, flush="sync") # C-status: one store document per tenant that every card reads instead of process-local state. # Schema, the one place the key names live: # STATUS_KEY -> {provider: {connectionId: { # "lastGoodSyncAt": ISO-8601 UTC, advanced only, by a publication or a mirror reading after one # "pass": {"startedAt": ISO, "endedAt": ISO | None, "outcome": one of PASS_OUTCOMES, # "detail": str, "durationSeconds": float | None, ...extra writer fields}, # "resync": {...the resync loop's state, JSON-safe..., "at": ISO}, # "<section>": {...} any other named section, merged by update_status, # "updatedAt": ISO}}} # Reserved provider "sync_worker" (connection "default"), written by the sync worker, never a connector: # "heartbeat": {pid, host, tenant, scope, at (epoch s), bootRestore} in the worker tenant's document # "catchUp": {connector: catch-up progress}, "mirrors": {connector: {snapshotId, at}} per tenant # "job": {id, submittedAt (ISO), image, tenant} the web's last HF sync job, per tenant # "schedule": {nextPassAt (ISO), at (ISO), reason, jobId} written by a --once job at its exit # "heartbeat.exitedAt" (epoch s) a --once job's last beat; that beat reads as no live worker # RULE: every writer merges one provider and connection entry. It never clobbers another # connector's entry, and a store that refuses is logged and answered False, never raised into a pass. STATUS_KEY = "connector_status_v1" PASS_OUTCOMES = ("running", "done", "continuing", "waiting", "held", "failed") _STATUS_RESERVED = ("lastGoodSyncAt", "updatedAt") def _utc_stamp(value=None): """A UTC ISO stamp from a datetime, epoch seconds or ISO text, or None when unreadable.""" if value is None: return _dt.datetime.now(_dt.timezone.utc).isoformat() try: if isinstance(value, _dt.datetime): parsed = value elif isinstance(value, (int, float)) and not isinstance(value, bool): parsed = _dt.datetime.fromtimestamp(float(value), _dt.timezone.utc) else: parsed = _dt.datetime.fromisoformat(str(value).strip().replace("Z", "+00:00")) if parsed.tzinfo is None: parsed = parsed.replace(tzinfo=_dt.timezone.utc) return parsed.astimezone(_dt.timezone.utc).isoformat() except (TypeError, ValueError, OverflowError, OSError): return None def _status_update(rt, provider, connection_id, change): """Apply `change(entry)` to one provider and connection entry; False when the store refuses.""" provider_key = _key(provider) if not provider_key: raise ValueError("provider is required") connection_key = _slot(connection_id) def _apply(current): document = dict(current) if isinstance(current, dict) else {} connections = document.get(provider_key) connections = dict(connections) if isinstance(connections, dict) else {} entry = connections.get(connection_key) entry = deepcopy(entry) if isinstance(entry, dict) else {} change(entry) entry["updatedAt"] = _utc_stamp() connections[connection_key] = entry document[provider_key] = connections return document try: rt.update(STATUS_KEY, _apply, flush="sync") return True except Exception as exc: # noqa: BLE001 - a status write never fails the pass that made it logging.getLogger(__name__).warning("Connector status write refused for %s/%s (%s)", provider_key, connection_key, type(exc).__name__) return False def _advance_good_sync(entry, at): stamp = _utc_stamp(at) if at is not None else None if stamp is None: return held = _utc_stamp(entry.get("lastGoodSyncAt")) if entry.get("lastGoodSyncAt") else None if held is None or _dt.datetime.fromisoformat(stamp) > _dt.datetime.fromisoformat(held): entry["lastGoodSyncAt"] = stamp def _status_document(rt, fresh): # `fresh` reads past this process's cache: the sync worker writes from another process. return (rt.get(STATUS_KEY, fresh=True) if fresh else rt.get(STATUS_KEY)) or {} def status(rt, provider, connection_id=None, *, fresh=False): """One connector's C-status entry as a copy, or None. A store read failure answers None.""" try: document = _status_document(rt, fresh) connections = document.get(_key(provider)) if isinstance(document, dict) else None entry = connections.get(_slot(connection_id)) if isinstance(connections, dict) else None return deepcopy(entry) if isinstance(entry, dict) else None except Exception: # noqa: BLE001 - status reads cannot take down their caller return None def status_all(rt, *, fresh=False): """Every provider's C-status entries as a copy; a store read failure answers {}.""" try: document = _status_document(rt, fresh) return {str(provider): deepcopy(connections) for provider, connections in document.items() if isinstance(connections, dict)} if isinstance(document, dict) else {} except Exception: # noqa: BLE001 - status reads cannot take down their caller return {} def last_good_sync(rt, provider, connection_id=None): """The stored lastGoodSyncAt as a UTC ISO stamp, or None.""" entry = status(rt, provider, connection_id) return _utc_stamp(entry.get("lastGoodSyncAt")) if entry and entry.get("lastGoodSyncAt") else None def record_good_sync(rt, provider, at=None, *, connection_id=None): """Advance lastGoodSyncAt to `at` (default now). An older stamp never moves it back.""" return _status_update(rt, provider, connection_id, lambda entry: _advance_good_sync(entry, at)) def record_pass_start(rt, provider, *, at=None, connection_id=None): """Stamp a pass as running. The previous pass's end is dropped with it.""" started = _utc_stamp(at) def change(entry): previous = entry.get("pass") if isinstance(entry.get("pass"), dict) else {} publication = {name: deepcopy(previous[name]) for name in ("tableSync", "held", "notOffered", "notOfferedTables") if name in previous} entry["pass"] = {**publication, "startedAt": started, "endedAt": None, "outcome": "running", "detail": "", "durationSeconds": None, "budgetSeconds": 1800} return _status_update(rt, provider, connection_id, change) def record_pass_end(rt, provider, outcome, *, at=None, detail="", last_good_sync_at=None, connection_id=None, extra=None): """Close the last pass with its outcome; `last_good_sync_at` advances lastGoodSyncAt.""" if outcome not in PASS_OUTCOMES or outcome == "running": raise ValueError(f"unknown pass outcome: {outcome}") ended = _utc_stamp(at) def change(entry): previous = entry.get("pass") if isinstance(entry.get("pass"), dict) else {} started = previous.get("startedAt") finished = previous.get("publishedAt") or ended duration = None if started and finished: duration = round((_dt.datetime.fromisoformat(finished) - _dt.datetime.fromisoformat(started)).total_seconds(), 3) publication = {name: deepcopy(previous[name]) for name in ("tableSync", "held", "notOffered", "notOfferedTables", "publishedAt") if name in previous} row = {**publication, "startedAt": started, "endedAt": finished, "outcome": outcome, "detail": str(detail or "")[:300], "durationSeconds": duration, "budgetSeconds": 1800} fixed = {"startedAt", "endedAt", "outcome", "detail", "durationSeconds", "budgetSeconds"} row.update({str(k): v for k, v in dict(extra or {}).items() if str(k) not in fixed}) entry["pass"] = row slot_date = str(row.get("slotDate") or started or ended or "")[:10] if slot_date: days = dict(entry.get("dailyPasses") or {}) days[slot_date] = {"date": slot_date, "minutes": round(duration / 60, 3) if duration is not None else None, "budgetMinutes": 30, "outcome": outcome} entry["dailyPasses"] = days if outcome != "failed": # Any pass that did not fail proves the credential works again. entry.pop("credentialHold", None) _advance_good_sync(entry, last_good_sync_at) return _status_update(rt, provider, connection_id, change) def credential_fingerprint(material): """The first 16 hex of sha256 over the credential material, or None when there is none. A list or tuple is joined line by line. The secret itself is never stored or returned. """ if isinstance(material, (list, tuple)): material = "\n".join(str(part) for part in material) if not isinstance(material, str) or not material: return None return hashlib.sha256(material.encode("utf-8")).hexdigest()[:16] def hold_credential(rt, provider, fingerprint, *, connection_id=None): """Record that the provider refused the credential with this fingerprint.""" if not fingerprint: return False at = _utc_stamp() def change(entry): entry["credentialHold"] = {"fingerprint": str(fingerprint), "at": at} return _status_update(rt, provider, connection_id, change) def credential_held(rt, provider, fingerprint, *, connection_id=None): """True when a scheduled pass must skip: the last pass failed on this same refused credential. An unknown fingerprint or an unreadable record answers False, so the pass runs. """ if not fingerprint: return False entry = status(rt, provider, connection_id, fresh=True) or {} hold = entry.get("credentialHold") last_pass = entry.get("pass") if isinstance(entry.get("pass"), dict) else {} return (isinstance(hold, dict) and hold.get("fingerprint") == fingerprint and last_pass.get("outcome") == "failed") def record_resync_state(rt, provider, state, *, connection_id=None): """Replace the connector's resync state with a JSON-safe copy of `state`.""" try: snapshot = json.loads(json.dumps(dict(state or {}), default=str)) except (TypeError, ValueError): snapshot = {} snapshot["at"] = _utc_stamp() def change(entry): entry["resync"] = snapshot return _status_update(rt, provider, connection_id, change) def update_status(rt, provider, section, fields, *, connection_id=None): """Merge `fields` into one named section of a connector's entry (first loads, heartbeats).""" name = _key(section) if not name or name in _STATUS_RESERVED: raise ValueError(f"section {section!r} is not writable through update_status") try: values = json.loads(json.dumps(dict(fields or {}), default=str)) except (TypeError, ValueError): values = {} def change(entry): current = entry.get(name) if isinstance(entry.get(name), dict) else {} entry[name] = {**current, **values} return _status_update(rt, provider, connection_id, change) def register_provider(provider, *, label, retry_hint): """Register provider metadata; runners remain in the API layer.""" provider_key = _key(provider) if not provider_key: raise ValueError("provider is required") _PROVIDERS[provider_key] = { "key": provider_key, "label": str(label or provider_key), "retryHint": str(retry_hint or ""), } def providers(): """Return a copy of registered provider metadata.""" return {key: dict(value) for key, value in _PROVIDERS.items()} register_provider("odoo", label="Odoo", retry_hint="Retry the Odoo database sync.") register_provider( "amazon_sp", label="Amazon", retry_hint="Use Sync now to retry the Amazon pull.", ) register_provider("meta_ads", label="Meta Ads", retry_hint="Use Sync now to retry the Meta Ads pull.") register_provider("amazon_ads", label="Amazon Ads", retry_hint="Use Sync now to retry the Amazon Ads pull.")
"""Tenant-and-connection scoped connector sync lifecycle state.
This data-layer module records what a provider runner did. It deliberately does not import or
call providers: the API layer owns runners while every connector shares this durable vocabulary.
"""
from __future__ import annotations
import datetime as _dt
import hashlib
import json
import logging
import threading
import time
import uuid
from copy import deepcopy
import core.store as store
STATE_KEY = "connector_sync_state"
STATES = ("queued", "syncing", "materializing", "ready", "partial", "error")
_ERROR_LIMIT = 50
_DEFAULT_CONNECTION = "default"
_PROVIDERS = {}
# Ordinary requests coalesce per provider/tenant. Explicit events retain separate durable receipts.
_SYNC_REQUESTS = {}
_SYNC_REQUEST_RUNTIMES = {}
_SYNC_NOT_BEFORE = {}
_SYNC_REQUEST_LOCK = threading.Lock()
_SYNC_WAKE = threading.Event()
_SYNC_DUE_TIMER = None
SYNC_REQUESTS_KEY = "connector_sync_requests_v1"
NOTIFICATION_TARGETING_CONTRACT = "amazon-table-events-v1"
# Ordinary requests the web process hands to the sync worker live in this member of the desk.
REMOTE_MEMBER = "ordinary"
REMOTE_KEEP_SECONDS = 3600
_REQUEST_LISTENERS = []
_REMOTE_IDS = {}
_EVENT_DEDUPE_SECONDS = 600
def _target_tables(provider, tenant, rt, tables, event_id):
from core.connector_spec import AMAZON_TABLES
known = {row.source.removeprefix("amzn_") for row in AMAZON_TABLES
if not row.native.startswith("derived:")}
if (provider != "amazon_sp" or rt is None or rt.key != tenant
or not isinstance(tables, list) or not tables
or any(not isinstance(table, str) or table not in known for table in tables)):
raise ValueError("amazon_notification_target_invalid")
if not isinstance(event_id, str) or not event_id.strip() or len(tables) != 1:
raise ValueError("Amazon notifications require one table and an event ID.")
return list(tables)
def _sync_request_document(current):
document = deepcopy(current) if current is not None else {}
if not isinstance(document, dict):
raise ValueError("The stored sync request desk is unavailable.")
for member in ("requests", "events", REMOTE_MEMBER):
document.setdefault(member, {})
if not isinstance(document[member], dict):
raise ValueError("The stored sync request desk is unavailable.")
return document
def _cache_sync_requests(rt, document):
"""Refresh this tenant's durable receipts under the desk lock without touching ordinary work."""
rows = {}
for identity, row in document["requests"].items():
if not isinstance(row, dict) or row.get("id") != identity or row.get("tenant") != rt.key:
raise ValueError("The stored sync receipt does not match its tenant or identity.")
_target_tables(row.get("provider"), rt.key, rt, row.get("tables"), row.get("eventId"))
rows[(row["provider"], rt.key, identity)] = deepcopy(row)
for key in list(_SYNC_REQUESTS):
if len(key) == 3 and key[1] == rt.key:
del _SYNC_REQUESTS[key]
_SYNC_REQUESTS.update(rows)
_SYNC_REQUEST_RUNTIMES[rt.key] = rt
def _wake_sync_requests(stamp, held=()):
_arm_sync_wake(stamp)
if any(row["state"] == "queued" and (row.get("notBefore") or 0) <= stamp
and row.get("requestedBy") not in held
for row in _SYNC_REQUESTS.values()):
_SYNC_WAKE.set()
def resume_held_requests(now=None):
"""Wake the resident loop for queued work a hold left waiting."""
with _SYNC_REQUEST_LOCK:
_wake_sync_requests(time.time() if now is None else now)
def restore_sync_requests(rt, *, now=None, requeue=True):
"""Restore durable events and requeue claims abandoned by a previous process.
requeue=False only reads: a process that never claims must not requeue another one's claim.
"""