Download platform/core/connector_sync.py from fsanyoto/loopable: direct link, hf CLI and curl.
- Browser
- Download file 57.7 kB
-
https://huggingface.co/spaces/fsanyoto/loopable/resolve/9291c0100f4e7b8da2efcbbf1ebdea72ee1663e7/platform/core/connector_sync.py
- Command line
-
hf download hf://spaces/fsanyoto/loopable@9291c0100f4e7b8da2efcbbf1ebdea72ee1663e7/platform/core/connector_sync.py
-
curl -L -o connector_sync.py https://huggingface.co/spaces/fsanyoto/loopable/resolve/9291c0100f4e7b8da2efcbbf1ebdea72ee1663e7/platform/core/connector_sync.py
57.7 kB
"""Tenant-and-connection scoped connector sync lifecycle state.
This data-layer module records what a provider runner did. It deliberately does not import or
call providers: the API layer owns runners while every connector shares this durable vocabulary.
"""
from __future__ import annotations
import datetime as _dt
import hashlib
import json
import logging
import threading
import time
import uuid
from copy import deepcopy
import core.store as store
STATE_KEY = "connector_sync_state"
STATES = ("queued", "syncing", "materializing", "ready", "partial", "error")
_ERROR_LIMIT = 50
_DEFAULT_CONNECTION = "default"
_PROVIDERS = {}
# Ordinary requests coalesce per provider/tenant. Explicit events retain separate durable receipts.
_SYNC_REQUESTS = {}
_SYNC_REQUEST_RUNTIMES = {}
_SYNC_NOT_BEFORE = {}
_SYNC_REQUEST_LOCK = threading.Lock()
_SYNC_WAKE = threading.Event()
_SYNC_DUE_TIMER = None
SYNC_REQUESTS_KEY = "connector_sync_requests_v1"
NOTIFICATION_TARGETING_CONTRACT = "amazon-table-events-v1"
# Ordinary requests the web process hands to the sync worker live in this member of the desk.
REMOTE_MEMBER = "ordinary"
REMOTE_KEEP_SECONDS = 3600
_REQUEST_LISTENERS = []
_REMOTE_IDS = {}
_EVENT_DEDUPE_SECONDS = 600
def _target_tables(provider, tenant, rt, tables, event_id):
from core.connector_spec import AMAZON_TABLES
known = {row.source.removeprefix("amzn_") for row in AMAZON_TABLES
if not row.native.startswith("derived:")}
if (provider != "amazon_sp" or rt is None or rt.key != tenant
or not isinstance(tables, list) or not tables
or any(not isinstance(table, str) or table not in known for table in tables)):
raise ValueError("amazon_notification_target_invalid")
if not isinstance(event_id, str) or not event_id.strip() or len(tables) != 1:
raise ValueError("Amazon notifications require one table and an event ID.")
return list(tables)
def _sync_request_document(current):
document = deepcopy(current) if current is not None else {}
if not isinstance(document, dict):
raise ValueError("The stored sync request desk is unavailable.")
for member in ("requests", "events", REMOTE_MEMBER):
document.setdefault(member, {})
if not isinstance(document[member], dict):
raise ValueError("The stored sync request desk is unavailable.")
return document
def _cache_sync_requests(rt, document):
"""Refresh this tenant's durable receipts under the desk lock without touching ordinary work."""
rows = {}
for identity, row in document["requests"].items():
if not isinstance(row, dict) or row.get("id") != identity or row.get("tenant") != rt.key:
raise ValueError("The stored sync receipt does not match its tenant or identity.")
_target_tables(row.get("provider"), rt.key, rt, row.get("tables"), row.get("eventId"))
rows[(row["provider"], rt.key, identity)] = deepcopy(row)
for key in list(_SYNC_REQUESTS):
if len(key) == 3 and key[1] == rt.key:
del _SYNC_REQUESTS[key]
_SYNC_REQUESTS.update(rows)
_SYNC_REQUEST_RUNTIMES[rt.key] = rt
def _wake_sync_requests(stamp, held=()):
_arm_sync_wake(stamp)
if any(row["state"] == "queued" and (row.get("notBefore") or 0) <= stamp
and row.get("requestedBy") not in held
for row in _SYNC_REQUESTS.values()):
_SYNC_WAKE.set()
def resume_held_requests(now=None):
"""Wake the resident loop for queued work a hold left waiting."""
with _SYNC_REQUEST_LOCK:
_wake_sync_requests(time.time() if now is None else now)
def restore_sync_requests(rt, *, now=None, requeue=True):
"""Restore durable events and requeue claims abandoned by a previous process.
requeue=False only reads: a process that never claims must not requeue another one's claim.
"""
stamp = time.time() if now is None else now
with _SYNC_REQUEST_LOCK:
document = _sync_request_document(rt.get(SYNC_REQUESTS_KEY, fresh=True))
abandoned = {identity for identity, row in document["requests"].items()
if requeue and row.get("state") == "running" and
_SYNC_REQUESTS.get((row.get("provider"), rt.key, identity), {}).get("state") != "running"}
if abandoned:
def restore(current):
doc = _sync_request_document(current)
for identity in abandoned:
row = doc["requests"].get(identity)
if row and row["state"] == "running":
row.update(state="queued", startedAt=None, claimToken="", restoredAt=stamp)
return doc
document = rt.update(SYNC_REQUESTS_KEY, restore, flush="sync")
_cache_sync_requests(rt, document)
_wake_sync_requests(stamp)
return [deepcopy(row) for row in document["requests"].values()]
def _arm_sync_wake(now):
"""Called under the desk lock. A future claim wakes the resident loop once, at its floor.
Bootstrap re-arms this disposable alarm from the provider's durable not-before marker.
The alarm only wakes the single writer; it neither queues work nor calls a provider.
"""
global _SYNC_DUE_TIMER
if _SYNC_DUE_TIMER is not None:
_SYNC_DUE_TIMER.cancel()
_SYNC_DUE_TIMER = None
due = [row["notBefore"] for row in _SYNC_REQUESTS.values()
if row["state"] == "queued" and row.get("notBefore") and row["notBefore"] > now]
if due:
_SYNC_DUE_TIMER = threading.Timer(min(due) - now, _SYNC_WAKE.set)
_SYNC_DUE_TIMER.daemon = True
_SYNC_DUE_TIMER.start()
def defer_sync(provider, tenant, not_before):
"""Install a provider's durable checkpoint boundary, including on process bootstrap.
The provider owns persistence and restores this boundary before re-queuing its work.
This module only claims intents; it never reads a provider mirror.
"""
with _SYNC_REQUEST_LOCK:
key = (provider, tenant)
if not_before is None:
_SYNC_NOT_BEFORE.pop(key, None)
else:
_SYNC_NOT_BEFORE[key] = float(not_before)
receipt = _SYNC_REQUESTS.get(key)
if receipt and receipt["state"] == "queued":
receipt["notBefore"] = _SYNC_NOT_BEFORE.get(key)
_arm_sync_wake(time.time())
if not not_before or float(not_before) <= time.time():
_SYNC_WAKE.set()
def sync_not_before(provider, tenant):
"""The provider's installed not-before floor as an epoch, or None."""
with _SYNC_REQUEST_LOCK:
return _SYNC_NOT_BEFORE.get((provider, tenant))
def _clear_sync_defer(provider, tenant):
"""Remove one obsolete provider hold under the desk lock, retaining explicit continuation waits."""
key = (provider, tenant)
held = _SYNC_NOT_BEFORE.pop(key, None)
receipt = _SYNC_REQUESTS.get(key)
if (held is not None and receipt and receipt["state"] == "queued"
and receipt.get("requestedBy") != "first load" and receipt.get("notBefore") == held):
receipt["notBefore"] = None
def clear_sync_defer(provider, tenant):
"""Release one provider hold after its producer adopts partition-specific waits."""
with _SYNC_REQUEST_LOCK:
_clear_sync_defer(provider, tenant)
_wake_sync_requests(time.time())
def register_request_listener(fn):
"""Call fn(receipt, connection_id) after each accepted ordinary request, outside the desk lock."""
if fn not in _REQUEST_LISTENERS:
_REQUEST_LISTENERS.append(fn)
def request_sync(provider, tenant, *, requested_by="", now=None, not_before=None,
rt=None, tables=None, event_id="", connection_id=None, wait_cause=""):
"""Queue ordinary work or durably acknowledge one tenant-bound Amazon event.
`wait_cause` is the safe sentence a receipt held by its not-before floor shows instead of the bare stamp.
"""
receipt = _request_sync(provider, tenant, requested_by=requested_by, now=now,
not_before=not_before, rt=rt, tables=tables, event_id=event_id,
wait_cause=wait_cause)
if tables is None and receipt.get("accepted"):
for listener in list(_REQUEST_LISTENERS):
listener(dict(receipt), connection_id)
return receipt
def _request_sync(provider, tenant, *, requested_by="", now=None, not_before=None,
rt=None, tables=None, event_id="", wait_cause=""):
if provider not in _PROVIDERS:
raise ValueError("This connector is not registered for sync.")
tenant = str(tenant or "").strip()
if not tenant:
raise ValueError("A tenant is required for sync.")
if tables is None and event_id:
raise ValueError("Amazon notifications require an explicit table target.")
if tables is not None:
targets = _target_tables(provider, tenant, rt, tables, event_id)
stamp, identity = time.time() if now is None else now, uuid.uuid4().hex
event_key = hashlib.sha256(json.dumps([provider, event_id], separators=(",", ":")).encode()).hexdigest()
receipt = {"id": identity, "provider": provider, "tenant": tenant, "tables": targets,
"eventId": event_id, "requestedBy": str(requested_by or ""),
"requestedAt": stamp, "notBefore": float(not_before) if not_before is not None else None,
"state": "queued", "startedAt": None, "finishedAt": None,
"detail": "", "accepted": False, "claimToken": ""}
def enqueue_event(current):
document = _sync_request_document(current)
event = document["events"].get(event_key)
if event and stamp < event["firstAcceptedAt"] + _EVENT_DEDUPE_SECONDS:
if event["requestId"] not in document["requests"]:
raise ValueError("The stored Amazon event receipt is unavailable.")
return document
document["requests"][identity] = deepcopy(receipt)
document["events"][event_key] = {"requestId": identity, "firstAcceptedAt": stamp}
return document
with _SYNC_REQUEST_LOCK:
document = rt.update(SYNC_REQUESTS_KEY, enqueue_event, flush="sync")
_cache_sync_requests(rt, document)
accepted_id = document["events"][event_key]["requestId"]
result = deepcopy(document["requests"][accepted_id])
_wake_sync_requests(stamp)
return {**result, "accepted": accepted_id == identity, "duplicate": accepted_id != identity}
with _SYNC_REQUEST_LOCK:
if provider == "amazon_sp" and requested_by != "first load":
# Amazon's history floor belongs to owed tables, never a loaded-table refresh.
_clear_sync_defer(provider, tenant)
previous = _SYNC_REQUESTS.get((provider, tenant))
replaces_amazon_history = (provider == "amazon_sp" and previous
and previous.get("requestedBy") == "first load"
and requested_by != "first load")
if previous and previous["state"] in ("queued", "running"):
if not (previous["state"] == "queued" and previous.get("requestedBy") == "first load"
and requested_by != "first load"):
return {**_request_wait(previous), "accepted": False}
receipt = {"id": uuid.uuid4().hex, "provider": provider, "tenant": tenant, "state": "queued",
"requestedAt": time.time() if now is None else now,
"notBefore": max(float(not_before or 0), _SYNC_NOT_BEFORE.get((provider, tenant), 0),
float(previous.get("notBefore") or 0) if previous
and previous["state"] == "queued" and not replaces_amazon_history else 0) or None,
"startedAt": None, "finishedAt": None, "detail": "",
"requestedBy": str(requested_by or ""), "accepted": True}
if requested_by == "first load" and previous and previous.get("firstLoad"):
# A continuation keeps the safe numeric progress visible while waiting for the writer.
receipt.update(detail=previous["detail"], firstLoad=dict(previous["firstLoad"]))
if wait_cause:
receipt.update(detail=str(wait_cause)[:300], waitCause=str(wait_cause)[:300])
_SYNC_REQUESTS[(provider, tenant)] = receipt
stamp = time.time() if now is None else now
_arm_sync_wake(stamp)
if (receipt.get("notBefore") or 0) <= stamp:
_SYNC_WAKE.set()
return _request_wait(receipt, now=stamp)
def _request_order(receipt):
"""Manual work precedes first-load continuations, with FIFO within each class."""
return receipt.get("requestedBy") == "first load", receipt.get("requestedAt") or 0
def _request_wait(receipt, *, now=None):
"""Describe a queued receipt's executing competitor or provider floor without claiming a start."""
result = deepcopy(receipt)
result.setdefault("waitingFor", None)
if receipt["state"] != "queued":
result["waitingFor"] = None
return result
running = [row for row in _SYNC_REQUESTS.values()
if row["tenant"] == receipt["tenant"] and row["state"] == "running"
and row.get("startedAt") is not None]
if running:
active = max(running, key=lambda row: row["startedAt"])
started = active["startedAt"]
identity = active.get("id") or f"{active['provider']}:{started}"
result["waitingFor"] = {"id": identity, "provider": active["provider"], "startedAt": started}
stamp = _dt.datetime.fromtimestamp(started, _dt.timezone.utc).isoformat()
result["detail"] = f"Waiting for {active['provider']} receipt {identity}, started at {stamp}."
elif result.get("waitingFor"):
return result
elif (receipt.get("notBefore") or 0) > (time.time() if now is None else now):
stamp = _dt.datetime.fromtimestamp(receipt["notBefore"], _dt.timezone.utc).isoformat()
result["detail"] = receipt.get("waitCause") or f"Waiting for the provider not-before floor at {stamp}."
elif result.get("detail", "").startswith("Waiting for "):
result["detail"] = ""
return result
def sync_request(provider, tenant):
"""Return a detached current receipt, or None."""
with _SYNC_REQUEST_LOCK:
rows = [row for row in _SYNC_REQUESTS.values()
if row["provider"] == provider and row["tenant"] == tenant]
receipt = next((row for row in rows if row["state"] == "running"), None)
if receipt is None:
queued = [row for row in rows if row["state"] == "queued"]
receipt = min(queued, key=_request_order) if queued else _SYNC_REQUESTS.get((provider, tenant))
if receipt is None:
return None
return {**_request_wait(receipt), "accepted": False}
def take_sync_requests(*, now=None, limit=None, exclude_providers=(), refresh_remote=False,
exclude_requested_by=()):
"""Claim manual work first, preserving provider floors, tenant claims and event siblings.
`exclude_requested_by` leaves those receipts queued and does not wake the loop for them.
"""
if refresh_remote:
with _SYNC_REQUEST_LOCK:
runtimes = list(_SYNC_REQUEST_RUNTIMES.values())
for rt in runtimes:
try:
poll_remote_requests(rt, now=now)
except Exception as exc:
logging.getLogger(__name__).warning("Sync desk refresh unavailable for %s (%s)",
rt.key, type(exc).__name__)
with _SYNC_REQUEST_LOCK:
_SYNC_WAKE.clear()
claimed = []
stamp = time.time() if now is None else now
busy = {(row["provider"], row["tenant"]) for row in _SYNC_REQUESTS.values() if row["state"] == "running"}
for receipt in sorted(list(_SYNC_REQUESTS.values()), key=_request_order):
if limit is not None and len(claimed) >= limit:
break
pair = (receipt["provider"], receipt["tenant"])
if (receipt["provider"] in exclude_providers or pair in busy or receipt["state"] != "queued"
or receipt.get("requestedBy") in exclude_requested_by
or max(receipt.get("notBefore") or 0, _SYNC_NOT_BEFORE.get(pair, 0)) > stamp):
continue
if receipt.get("tables") is not None:
rt = _SYNC_REQUEST_RUNTIMES[receipt["tenant"]]
identity, token = receipt["id"], uuid.uuid4().hex
def claim_event(current):
document = _sync_request_document(current)
rows = [row for row in document["requests"].values()
if (row["provider"], row["tenant"]) == pair]
if any(row["state"] == "running" for row in rows):
return document
eligible = [row for row in rows if row["state"] == "queued"
and (row.get("notBefore") or 0) <= stamp]
row = min(eligible, key=_request_order) if eligible else None
if row is not None and row["id"] == identity:
row.update(state="running", startedAt=stamp, claimToken=token, accepted=False)
return document
document = rt.update(SYNC_REQUESTS_KEY, claim_event, flush="sync")
_cache_sync_requests(rt, document)
receipt = document["requests"].get(identity, {})
if receipt.get("claimToken") != token:
continue
else:
receipt.update(state="running", startedAt=stamp, accepted=False)
receipt["waitingFor"] = None
if receipt.get("detail", "").startswith("Waiting for "):
receipt["detail"] = ""
claimed.append(deepcopy(receipt))
busy.add(pair)
_wake_sync_requests(stamp, exclude_requested_by)
return claimed
def finish_sync_request(provider, tenant, *, ok, detail="", now=None, first_load=None, request_id=None):
"""Finish a claimed intent. The API runner supplies a safe cause sentence, never raw errors."""
with _SYNC_REQUEST_LOCK:
receipt = next((row for row in _SYNC_REQUESTS.values() if row["provider"] == provider
and row["tenant"] == tenant and row["state"] == "running"
and (request_id is None or row.get("id") == request_id)), None)
if receipt is None:
return None
stamp = time.time() if now is None else now
completion = {"state": "done" if ok else "failed", "detail": str(detail or "")[:300],
"finishedAt": stamp, "accepted": False}
if first_load is not None:
completion["firstLoad"] = dict(first_load)
if receipt.get("tables") is not None:
rt = _SYNC_REQUEST_RUNTIMES[tenant]
identity, token = receipt["id"], receipt["claimToken"]
def finish_event(current):
document = _sync_request_document(current)
row = document["requests"].get(identity)
if row and row["state"] == "running" and row.get("claimToken") == token:
row.update(completion)
return document
document = rt.update(SYNC_REQUESTS_KEY, finish_event, flush="sync")
_cache_sync_requests(rt, document)
receipt = document["requests"].get(identity, {})
if receipt.get("claimToken") != token or receipt.get("finishedAt") != stamp:
return None
else:
receipt.update(completion)
result = deepcopy(receipt)
result["waitingFor"] = None
_wake_sync_requests(stamp)
connector = {"amazon_sp": "amazon", "meta_ads": "meta"}.get(provider)
if connector:
try:
from core.sync_events import emit
except ImportError:
pass # C20: lane J supplies this module at combine.
else:
try:
emit(tenant, connector, bool(ok), result["detail"],
_dt.datetime.fromtimestamp(result["finishedAt"], _dt.timezone.utc).isoformat())
except Exception:
logging.getLogger(__name__).exception("Sync completion listener failed")
return result
def sync_wake_event():
"""The event the resident thread waits on between passes."""
return _SYNC_WAKE
def queue_remote_request(rt, receipt, connection_id=None, *, now=None):
"""Web side: durably hand one accepted ordinary request to the sync worker process."""
stamp = time.time() if now is None else now
identity = receipt.get("id") or uuid.uuid4().hex
entry = {"id": identity, "provider": receipt["provider"], "tenant": rt.key,
"requestedBy": str(receipt.get("requestedBy") or ""),
"requestedAt": float(receipt.get("requestedAt") or stamp),
"notBefore": receipt.get("notBefore"),
"connectionId": str(connection_id) if connection_id else None,
"state": "queued", "startedAt": None, "finishedAt": None, "detail": ""}
def add(current):
document = _sync_request_document(current)
document[REMOTE_MEMBER][identity] = entry
return document
rt.update(SYNC_REQUESTS_KEY, add, flush="sync")
with _SYNC_REQUEST_LOCK:
_REMOTE_IDS[(receipt["provider"], rt.key)] = identity
return identity
def fail_remote_request(provider, tenant, detail, *, now=None):
"""Web side: a hand-off that could not be saved finishes its receipt instead of hanging."""
with _SYNC_REQUEST_LOCK:
receipt = _SYNC_REQUESTS.get((provider, tenant))
if receipt and receipt["state"] == "queued":
receipt.update(state="failed", detail=str(detail)[:300],
finishedAt=time.time() if now is None else now)
def _remote_progress(entry, receipt):
"""The worker receipt's progress for one entry, or None when that run does not cover it."""
started = receipt.get("startedAt") if receipt else None
if (started is None or entry["requestedAt"] > started
or receipt["state"] not in ("running", "done", "failed")
or (entry.get("requestedBy") != "first load" and receipt.get("requestedBy") == "first load")):
return None
progress = {"state": receipt["state"], "startedAt": started,
"finishedAt": receipt.get("finishedAt"), "detail": receipt.get("detail") or "",
"waitingFor": None}
if receipt.get("firstLoad"):
progress["firstLoad"] = dict(receipt["firstLoad"])
return progress
def poll_remote_requests(rt, *, now=None):
"""Worker side: admit queued hand-offs, publish their progress, prune old ones.
An entry leaves the queue only when a run that started after it was requested finishes.
Returns the entries admitted as new local receipts.
"""
stamp = time.time() if now is None else now
admitted, changes, pruned = [], {}, []
with _SYNC_REQUEST_LOCK:
document = _sync_request_document(rt.get(SYNC_REQUESTS_KEY, fresh=True))
for identity, entry in sorted(document[REMOTE_MEMBER].items(), key=lambda item: _request_order(item[1])):
key = (entry.get("provider"), rt.key)
if entry.get("tenant") != rt.key or key[0] not in _PROVIDERS:
continue
if entry["state"] in ("done", "failed"):
if stamp - float(entry.get("finishedAt") or 0) > REMOTE_KEEP_SECONDS:
pruned.append(identity)
continue
manual_amazon = key[0] == "amazon_sp" and entry.get("requestedBy") != "first load"
if manual_amazon:
_clear_sync_defer(*key)
receipt = _SYNC_REQUESTS.get(key)
# Promote under the same lock as claims, so a continuation cannot win a manual hand-off race.
if (receipt and receipt["state"] == "queued" and receipt.get("requestedBy") == "first load"
and entry.get("requestedBy") != "first load"):
receipt.update(id=identity, requestedBy=entry.get("requestedBy") or "",
requestedAt=entry["requestedAt"],
connectionId=entry.get("connectionId"),
notBefore=max(0 if manual_amazon else float(receipt.get("notBefore") or 0),
float(entry.get("notBefore") or 0),
_SYNC_NOT_BEFORE.get(key, 0)) or None)
admitted.append(deepcopy(entry))
progress = _remote_progress(entry, receipt)
if progress is not None:
if any(entry.get(name) != value for name, value in progress.items()):
changes[identity] = progress
continue
# A "running" entry with no local run was abandoned by a worker restart: admit it again.
if (entry["state"] in ("queued", "running")
and not (receipt and receipt["state"] in ("queued", "running"))):
local = {"id": identity, "provider": key[0], "tenant": rt.key, "state": "queued",
"requestedAt": entry["requestedAt"],
"notBefore": max(float(entry.get("notBefore") or 0),
_SYNC_NOT_BEFORE.get(key, 0)) or None,
"startedAt": None, "finishedAt": None, "detail": "",
"connectionId": entry.get("connectionId"),
"requestedBy": entry.get("requestedBy") or "", "accepted": True}
if receipt and receipt.get("firstLoad") and local["requestedBy"] == "first load":
local.update(detail=receipt["detail"], firstLoad=dict(receipt["firstLoad"]))
_SYNC_REQUESTS[key] = local
_SYNC_REQUEST_RUNTIMES.setdefault(rt.key, rt)
admitted.append(deepcopy(entry))
receipt = local
waiting = _request_wait({**entry, "state": "queued", "startedAt": None,
"finishedAt": None, "waitingFor": None,
"notBefore": max(float(entry.get("notBefore") or 0),
_SYNC_NOT_BEFORE.get(key, 0),
float(receipt.get("notBefore") or 0)
if receipt and receipt["state"] == "queued" else 0) or None},
now=stamp)
fields = {name: waiting.get(name) for name in
("state", "startedAt", "finishedAt", "notBefore", "detail", "waitingFor")}
if any(entry.get(name) != value for name, value in fields.items()):
changes[identity] = fields
if changes or pruned:
def publish(current):
doc = _sync_request_document(current)
for identity, progress in changes.items():
if identity in doc[REMOTE_MEMBER]:
doc[REMOTE_MEMBER][identity].update(progress)
for identity in pruned:
doc[REMOTE_MEMBER].pop(identity, None)
return doc
rt.update(SYNC_REQUESTS_KEY, publish, flush="sync")
if admitted:
_wake_sync_requests(stamp)
return admitted
def apply_remote_progress(rt):
"""Web side: copy the worker's published progress onto this process's receipts."""
document = _sync_request_document(rt.get(SYNC_REQUESTS_KEY, fresh=True))
applied = 0
with _SYNC_REQUEST_LOCK:
for identity, entry in document[REMOTE_MEMBER].items():
key = (entry.get("provider"), rt.key)
receipt = _SYNC_REQUESTS.get(key)
if _REMOTE_IDS.get(key) != identity or receipt is None:
continue
fields = {name: entry.get(name) for name in
("state", "startedAt", "finishedAt", "detail", "notBefore", "waitingFor")}
if entry.get("firstLoad"):
fields["firstLoad"] = dict(entry["firstLoad"])
if any(receipt.get(k) != v for k, v in fields.items()):
receipt.update(fields)
applied += 1
# Event receipts the worker claimed or finished, refreshed without requeueing anything.
_cache_sync_requests(rt, document)
return applied
# One receipt per immutable database identity. No credentials, session, or source values.
JOBS_KEY = "connector_materialization_jobs"
MAX_ATTEMPTS = 3
LEASE_SECONDS = 300
LIFECYCLE_STATES = ("configured", "catalogue-ready", "queued", "syncing", "ready", "failed")
def job_key(provider, connection_id, database_key):
return json.dumps([_key(provider), _slot(connection_id), _key(database_key)],
separators=(",", ":"))
def _job_public(row):
answer = {k: v for k, v in row.items() if k != "leaseToken"}
# Older status readers only know error. Keep one persisted cause and project it there.
if row.get("state") == "queued" and row.get("stall"):
answer["error"] = row["stall"]["cause"]
answer["retryable"] = (row.get("state") == "failed"
and int(row.get("attempts", 0)) < MAX_ATTEMPTS)
answer["maxAttempts"] = MAX_ATTEMPTS
if row.get("state") == "failed" and not answer["retryable"]:
answer["error"] = "Sync attempts exhausted. Check the connection and update its settings."
return answer
def jobs(rt, provider=None, connection_id=None):
"""Persisted status, including receipts left by a previous process."""
document = rt.get(JOBS_KEY, fresh=True) or {}
return [_job_public(row) for row in document.values()
if isinstance(row, dict)
and (provider is None or row.get("provider") == provider)
and (connection_id is None or row.get("connectionId") == _slot(connection_id))]
def job_summary(receipts):
"""Aggregate only durable facts; a retry of ready work must never claim it is queued."""
if not receipts:
return None
states = {row.get("state") for row in receipts}
state = next((value for value in ("syncing", "queued", "failed", "ready", "catalogue-ready", "configured")
if value in states), "configured")
errors = [row["error"] for row in receipts if row.get("error")]
stamp = max(row.get("at", 0) for row in receipts)
return {"state": state, "at": _dt.datetime.fromtimestamp(stamp, _dt.timezone.utc).isoformat(),
"retryable": any(row.get("retryable") for row in receipts),
"databases": [row["databaseKey"] for row in receipts if row.get("state") == "ready"],
"detail": f"Database sync {state}", "errors": errors, "errorsTotal": len(errors)}
def configuration_revision(provider, connection_id, database_key, model, field_choices):
"""Fingerprint this database's field choices, never rows or sibling configuration."""
identity = database_identity(provider, connection_id, database_key, model)
selected = []
for key, value in (field_choices or {}).items():
scoped = parse_field_key(key)
if scoped and scoped[:4] == (identity["provider"], identity["connectionId"],
identity["databaseKey"], identity["model"]):
selected.append((scoped[4], value is True))
return hashlib.sha256(json.dumps(sorted(selected), separators=(",", ":")).encode()).hexdigest()
def enqueue(rt, provider, connection_id, database_key, *, model="", retry=False,
changed=False, revision=None, now=None):
"""A Save is idempotent. Failed work requires an explicit, bounded retry."""
identity = database_identity(provider, connection_id, database_key, model)
key = job_key(provider, connection_id, database_key)
stamp, receipt = time.time() if now is None else now, uuid.uuid4().hex
def apply(current):
doc = dict(current or {})
previous = doc.get(key) or {}
updated = changed or (revision is not None and previous.get("revision") != revision)
if not updated and previous.get("state") in ("queued", "syncing", "ready"):
return doc
attempts = int(previous.get("attempts", 0))
if not updated and previous.get("state") == "failed" and (not retry or attempts >= MAX_ATTEMPTS):
return doc
doc[key] = {**identity, "id": receipt, "state": "queued", "at": stamp,
"attempts": attempts if retry and not updated else 0, "leaseUntil": 0,
"revision": revision,
"leaseToken": "", "error": "", "workspaceChanged": False}
return doc
landed = rt.update(JOBS_KEY, apply, flush="sync")
return _job_public(landed[key])
def stall(rt, candidate, cause, *, now=None):
"""Record WHY queued work cannot start, without consuming an attempt."""
key = job_key(candidate["provider"], candidate["connectionId"], candidate["databaseKey"])
stamp = time.time() if now is None else now
sentence = str(cause or "")[:300]
def eligible(row):
return (row.get("id") == candidate.get("id") and
(row.get("state") == "queued" or
row.get("state") == "syncing" and row.get("leaseUntil", 0) <= stamp))
previous = (rt.get(JOBS_KEY, fresh=True) or {}).get(key) or {}
# Avoid even submitting a store update on the unchanged 15-second drain pass.
if not eligible(previous) or (previous.get("stall") or {}).get("cause") == sentence:
return _job_public(previous)
def apply(current):
doc = dict(current or {})
row = dict(doc.get(key) or {})
if not eligible(row) or (row.get("stall") or {}).get("cause") == sentence:
return doc
row.update(state="queued", leaseUntil=0, leaseToken="",
stall={"at": stamp, "cause": sentence})
doc[key] = row
return doc
landed = rt.update(JOBS_KEY, apply, flush="sync")
return _job_public(landed.get(key) or {})
def claim(rt, candidate, *, now=None):
"""Atomically claim queued or abandoned work; replay callbacks have no side effects."""
key = job_key(candidate["provider"], candidate["connectionId"], candidate["databaseKey"])
stamp, token = time.time() if now is None else now, uuid.uuid4().hex
def apply(current):
doc = dict(current or {})
row = dict(doc.get(key) or {})
if row.get("id") != candidate.get("id"):
return doc
if row.get("state") != "queued" and not (
row.get("state") == "syncing" and row.get("leaseUntil", 0) <= stamp):
return doc
row.pop("stall", None)
if int(row.get("attempts", 0)) >= MAX_ATTEMPTS:
row.update(state="failed", at=stamp, leaseToken="", leaseUntil=0,
error="Sync attempts exhausted. Check the connection and update its settings.")
else:
row.update(state="syncing", at=stamp, attempts=int(row.get("attempts", 0)) + 1,
leaseToken=token, leaseUntil=stamp + LEASE_SECONDS)
doc[key] = row
return doc
landed = rt.update(JOBS_KEY, apply, flush="sync")
row = landed.get(key) or {}
return dict(row) if row.get("leaseToken") == token else None
def owns(rt, job, *, now=None):
row = (rt.get(JOBS_KEY, fresh=True) or {}).get(job_key(
job["provider"], job["connectionId"], job["databaseKey"])) or {}
return (row.get("id") == job.get("id") and row.get("state") == "syncing"
and row.get("leaseToken") == job.get("leaseToken")
and row.get("leaseUntil", 0) > (time.time() if now is None else now))
def progress(rt, job, *, rows, total):
"""Record published rows only while this worker still owns its unexpired lease."""
key = job_key(job["provider"], job["connectionId"], job["databaseKey"])
stamp = time.time()
def apply(current):
doc = dict(current or {})
row = dict(doc.get(key) or {})
if (row.get("id") != job.get("id") or row.get("state") != "syncing"
or row.get("leaseToken") != job.get("leaseToken")
or row.get("leaseUntil", 0) <= stamp):
return doc
row["progress"] = {"rows": rows, "total": total, "at": stamp}
doc[key] = row
return doc
landed = rt.update(JOBS_KEY, apply, flush="sync")
return _job_public(landed.get(key) or {})
def finish(rt, job, *, success=False, renew=False, interrupted=False, cause="", now=None):
"""A stale worker cannot acknowledge, renew, or overwrite a newer receipt."""
key = job_key(job["provider"], job["connectionId"], job["databaseKey"])
stamp = time.time() if now is None else now
def apply(current):
doc = dict(current or {})
row = dict(doc.get(key) or {})
if (row.get("id") != job.get("id") or row.get("state") != "syncing"
or row.get("leaseToken") != job.get("leaseToken")
or row.get("leaseUntil", 0) <= stamp):
return doc
if renew:
row["leaseUntil"] = stamp + LEASE_SECONDS
elif interrupted:
# Disconnect suspends the current attempt, preserving earlier failures and identity.
row.update(state="queued", at=stamp, attempts=max(0, int(row.get("attempts", 0)) - 1),
leaseUntil=0, leaseToken="", workspaceChanged=False, error="")
else:
row.update(state="ready" if success else "failed", at=stamp,
leaseUntil=0, leaseToken="", workspaceChanged=bool(success),
error="" if success else (str(cause or "")[:300] or
"Rows could not be synced. Check the connection and retry."))
doc[key] = row
return doc
landed = rt.update(JOBS_KEY, apply, flush="sync")
return _job_public(landed.get(key) or {})
def cancel(rt, provider, connection_id, database_key):
key = job_key(provider, connection_id, database_key)
stamp = time.time()
def apply(current):
doc = dict(current or {})
if key in doc:
doc[key] = {**doc[key], "state": "configured", "at": stamp,
"leaseToken": "", "leaseUntil": 0, "workspaceChanged": False,
"error": ""}
doc[key].pop("stall", None)
doc[key].pop("progress", None)
return doc
rt.update(JOBS_KEY, apply, flush="sync")
def _st(st):
return st if st is not None else store
def _key(value):
return str(value or "").strip()
def _slot(connection_id):
return _key(connection_id) or _DEFAULT_CONNECTION
def _target(rt, st):
return st if st is not None else rt
def database_identity(provider, connection_id, database_key, model=""):
"""Source-neutral identity; labels and sync state never participate in a database key."""
if not _key(provider) or not _key(database_key):
raise ValueError("provider and database key are required")
return {"provider": _key(provider), "connectionId": _slot(connection_id),
"databaseKey": _key(database_key), "model": _key(model)}
def field_key(provider, connection_id, database_key, model, raw_name):
"""An opaque, collision-free key for one source field in one database."""
identity = database_identity(provider, connection_id, database_key, model)
if not identity["model"] or not _key(raw_name):
raise ValueError("model and raw field name are required")
return json.dumps([identity["provider"], identity["connectionId"],
identity["databaseKey"], identity["model"], _key(raw_name)],
separators=(",", ":"), ensure_ascii=True)
def parse_field_key(value):
"""Read a scoped field key, or return None for a legacy model.field key."""
try:
parts = json.loads(value)
except (TypeError, ValueError):
return None
if not isinstance(parts, list) or len(parts) != 5 or not all(
isinstance(part, str) and part for part in parts):
return None
return (*parts,)
def _public(provider, connection_id, row):
if not isinstance(row, dict):
return None
answer = dict(row)
answer["provider"] = provider
answer["connectionId"] = connection_id
# RULE: retryability follows the state. Storing it would let the two answers drift.
answer["retryable"] = answer.get("state") in ("error", "partial")
return answer
def record(rt, provider, connection_id, state, *, detail="", errors=(), databases=(), st=None):
"""Record one lifecycle transition and return its defensive public representation."""
provider_key = _key(provider)
if not provider_key:
raise ValueError("provider is required")
if state not in STATES:
# RULE: unknown states are refused. A free-form status eventually stores a typo that the
# UI cannot render as one of C6's six observable states.
raise ValueError(f"unknown connector sync state: {state}")
connection_key = _slot(connection_id)
all_errors = [str(value) for value in (errors or ())]
visible_errors = all_errors[:_ERROR_LIMIT]
now = _dt.datetime.now(_dt.timezone.utc).isoformat()
written = {}
def _apply(current):
document = dict(current or {})
providers_doc = dict(document.get(provider_key) or {})
previous = providers_doc.get(connection_key)
previous_state = previous.get("state") if isinstance(previous, dict) else None
row = {
"state": state,
# RULE: stamp every transition and name its predecessor. A resting state alone cannot
# tell the UI whether work advanced, retried, or regressed.
"at": now,
"from": previous_state,
"detail": str(detail or ""),
# RULE: partial and error stay distinct. Partial exposes databases usable now;
# error means no usable set was produced.
"errors": visible_errors,
# RULE: retain up to 50 errors and report the full count. A storage limit must be
# observable rather than silently turning additional failures into zero information.
"errorsTotal": len(all_errors),
# These are governed table keys, not copied provider rows. In Odoo, row_limit == 0
# keeps 963,783 GL lines out of the editable user_tables document.
"databases": [str(value) for value in (databases or ()) if str(value)],
}
providers_doc[connection_key] = row
# RULE: key state by provider and connection. The measured legacy defect is that
# odoo_sync_status is one tenant document, so a second Odoo connection overwrites the
# first one's status. The default slot matches Odoo's existing catalog convention.
document[provider_key] = providers_doc
written.update(row)
return document
_st(_target(rt, st)).update(STATE_KEY, _apply, flush="sync")
return _public(provider_key, connection_key, written)
def read(rt, provider, connection_id=None, st=None):
"""Read one connection defensively; store failures mean no recorded state, never a 500."""
try:
provider_key = _key(provider)
connection_key = _slot(connection_id)
document = _st(_target(rt, st)).get(STATE_KEY) or {}
providers_doc = document.get(provider_key) if isinstance(document, dict) else None
row = providers_doc.get(connection_key) if isinstance(providers_doc, dict) else None
return _public(provider_key, connection_key, row)
except Exception: # noqa: BLE001 - status reads cannot take down their caller
return None
def read_all(rt, st=None):
"""Read every provider and connection defensively."""
try:
document = _st(_target(rt, st)).get(STATE_KEY) or {}
answer = {}
for provider_key, connections in document.items():
if not isinstance(connections, dict):
continue
public_connections = {}
for connection_key, row in connections.items():
public = _public(str(provider_key), str(connection_key), row)
if public is not None:
public_connections[str(connection_key)] = public
if public_connections:
answer[str(provider_key)] = public_connections
return answer
except Exception: # noqa: BLE001 - status reads cannot take down their caller
return {}
def clear(rt, provider, connection_id, st=None):
"""Remove only the named provider/connection state."""
provider_key = _key(provider)
connection_key = _slot(connection_id)
def _drop(current):
document = dict(current or {})
connections = dict(document.get(provider_key) or {})
connections.pop(connection_key, None)
if connections:
document[provider_key] = connections
else:
document.pop(provider_key, None)
return document
_st(_target(rt, st)).update(STATE_KEY, _drop, flush="sync")
# C-status: one store document per tenant that every card reads instead of process-local state.
# Schema, the one place the key names live:
# STATUS_KEY -> {provider: {connectionId: {
# "lastGoodSyncAt": ISO-8601 UTC, advanced only, by a publication or a mirror reading after one
# "pass": {"startedAt": ISO, "endedAt": ISO | None, "outcome": one of PASS_OUTCOMES,
# "detail": str, "durationSeconds": float | None, ...extra writer fields},
# "resync": {...the resync loop's state, JSON-safe..., "at": ISO},
# "<section>": {...} any other named section, merged by update_status,
# "updatedAt": ISO}}}
# Reserved provider "sync_worker" (connection "default"), written by the sync worker, never a connector:
# "heartbeat": {pid, host, tenant, scope, at (epoch s), bootRestore} in the worker tenant's document
# "catchUp": {connector: catch-up progress}, "mirrors": {connector: {snapshotId, at}} per tenant
# "job": {id, submittedAt (ISO), image, tenant} the web's last HF sync job, per tenant
# "schedule": {nextPassAt (ISO), at (ISO), reason, jobId} written by a --once job at its exit
# "heartbeat.exitedAt" (epoch s) a --once job's last beat; that beat reads as no live worker
# RULE: every writer merges one provider and connection entry. It never clobbers another
# connector's entry, and a store that refuses is logged and answered False, never raised into a pass.
STATUS_KEY = "connector_status_v1"
PASS_OUTCOMES = ("running", "done", "continuing", "waiting", "held", "failed")
_STATUS_RESERVED = ("lastGoodSyncAt", "updatedAt")
def _utc_stamp(value=None):
"""A UTC ISO stamp from a datetime, epoch seconds or ISO text, or None when unreadable."""
if value is None:
return _dt.datetime.now(_dt.timezone.utc).isoformat()
try:
if isinstance(value, _dt.datetime):
parsed = value
elif isinstance(value, (int, float)) and not isinstance(value, bool):
parsed = _dt.datetime.fromtimestamp(float(value), _dt.timezone.utc)
else:
parsed = _dt.datetime.fromisoformat(str(value).strip().replace("Z", "+00:00"))
if parsed.tzinfo is None:
parsed = parsed.replace(tzinfo=_dt.timezone.utc)
return parsed.astimezone(_dt.timezone.utc).isoformat()
except (TypeError, ValueError, OverflowError, OSError):
return None
def _status_update(rt, provider, connection_id, change):
"""Apply `change(entry)` to one provider and connection entry; False when the store refuses."""
provider_key = _key(provider)
if not provider_key:
raise ValueError("provider is required")
connection_key = _slot(connection_id)
def _apply(current):
document = dict(current) if isinstance(current, dict) else {}
connections = document.get(provider_key)
connections = dict(connections) if isinstance(connections, dict) else {}
entry = connections.get(connection_key)
entry = deepcopy(entry) if isinstance(entry, dict) else {}
change(entry)
entry["updatedAt"] = _utc_stamp()
connections[connection_key] = entry
document[provider_key] = connections
return document
try:
rt.update(STATUS_KEY, _apply, flush="sync")
return True
except Exception as exc: # noqa: BLE001 - a status write never fails the pass that made it
logging.getLogger(__name__).warning("Connector status write refused for %s/%s (%s)",
provider_key, connection_key, type(exc).__name__)
return False
def _advance_good_sync(entry, at):
stamp = _utc_stamp(at) if at is not None else None
if stamp is None:
return
held = _utc_stamp(entry.get("lastGoodSyncAt")) if entry.get("lastGoodSyncAt") else None
if held is None or _dt.datetime.fromisoformat(stamp) > _dt.datetime.fromisoformat(held):
entry["lastGoodSyncAt"] = stamp
def _status_document(rt, fresh):
# `fresh` reads past this process's cache: the sync worker writes from another process.
return (rt.get(STATUS_KEY, fresh=True) if fresh else rt.get(STATUS_KEY)) or {}
def status(rt, provider, connection_id=None, *, fresh=False):
"""One connector's C-status entry as a copy, or None. A store read failure answers None."""
try:
document = _status_document(rt, fresh)
connections = document.get(_key(provider)) if isinstance(document, dict) else None
entry = connections.get(_slot(connection_id)) if isinstance(connections, dict) else None
return deepcopy(entry) if isinstance(entry, dict) else None
except Exception: # noqa: BLE001 - status reads cannot take down their caller
return None
def status_all(rt, *, fresh=False):
"""Every provider's C-status entries as a copy; a store read failure answers {}."""
try:
document = _status_document(rt, fresh)
return {str(provider): deepcopy(connections) for provider, connections in document.items()
if isinstance(connections, dict)} if isinstance(document, dict) else {}
except Exception: # noqa: BLE001 - status reads cannot take down their caller
return {}
def last_good_sync(rt, provider, connection_id=None):
"""The stored lastGoodSyncAt as a UTC ISO stamp, or None."""
entry = status(rt, provider, connection_id)
return _utc_stamp(entry.get("lastGoodSyncAt")) if entry and entry.get("lastGoodSyncAt") else None
def record_good_sync(rt, provider, at=None, *, connection_id=None):
"""Advance lastGoodSyncAt to `at` (default now). An older stamp never moves it back."""
return _status_update(rt, provider, connection_id, lambda entry: _advance_good_sync(entry, at))
def record_pass_start(rt, provider, *, at=None, connection_id=None):
"""Stamp a pass as running. The previous pass's end is dropped with it."""
started = _utc_stamp(at)
def change(entry):
previous = entry.get("pass") if isinstance(entry.get("pass"), dict) else {}
publication = {name: deepcopy(previous[name]) for name in
("tableSync", "held", "notOffered", "notOfferedTables") if name in previous}
entry["pass"] = {**publication, "startedAt": started, "endedAt": None, "outcome": "running",
"detail": "", "durationSeconds": None, "budgetSeconds": 1800}
return _status_update(rt, provider, connection_id, change)
def record_pass_end(rt, provider, outcome, *, at=None, detail="", last_good_sync_at=None,
connection_id=None, extra=None):
"""Close the last pass with its outcome; `last_good_sync_at` advances lastGoodSyncAt."""
if outcome not in PASS_OUTCOMES or outcome == "running":
raise ValueError(f"unknown pass outcome: {outcome}")
ended = _utc_stamp(at)
def change(entry):
previous = entry.get("pass") if isinstance(entry.get("pass"), dict) else {}
started = previous.get("startedAt")
finished = previous.get("publishedAt") or ended
duration = None
if started and finished:
duration = round((_dt.datetime.fromisoformat(finished)
- _dt.datetime.fromisoformat(started)).total_seconds(), 3)
publication = {name: deepcopy(previous[name]) for name in
("tableSync", "held", "notOffered", "notOfferedTables", "publishedAt") if name in previous}
row = {**publication, "startedAt": started, "endedAt": finished, "outcome": outcome,
"detail": str(detail or "")[:300], "durationSeconds": duration,
"budgetSeconds": 1800}
fixed = {"startedAt", "endedAt", "outcome", "detail", "durationSeconds", "budgetSeconds"}
row.update({str(k): v for k, v in dict(extra or {}).items() if str(k) not in fixed})
entry["pass"] = row
slot_date = str(row.get("slotDate") or started or ended or "")[:10]
if slot_date:
days = dict(entry.get("dailyPasses") or {})
days[slot_date] = {"date": slot_date,
"minutes": round(duration / 60, 3) if duration is not None else None,
"budgetMinutes": 30, "outcome": outcome}
entry["dailyPasses"] = days
if outcome != "failed":
# Any pass that did not fail proves the credential works again.
entry.pop("credentialHold", None)
_advance_good_sync(entry, last_good_sync_at)
return _status_update(rt, provider, connection_id, change)
def credential_fingerprint(material):
"""The first 16 hex of sha256 over the credential material, or None when there is none.
A list or tuple is joined line by line. The secret itself is never stored or returned.
"""
if isinstance(material, (list, tuple)):
material = "\n".join(str(part) for part in material)
if not isinstance(material, str) or not material:
return None
return hashlib.sha256(material.encode("utf-8")).hexdigest()[:16]
def hold_credential(rt, provider, fingerprint, *, connection_id=None):
"""Record that the provider refused the credential with this fingerprint."""
if not fingerprint:
return False
at = _utc_stamp()
def change(entry):
entry["credentialHold"] = {"fingerprint": str(fingerprint), "at": at}
return _status_update(rt, provider, connection_id, change)
def credential_held(rt, provider, fingerprint, *, connection_id=None):
"""True when a scheduled pass must skip: the last pass failed on this same refused credential.
An unknown fingerprint or an unreadable record answers False, so the pass runs.
"""
if not fingerprint:
return False
entry = status(rt, provider, connection_id, fresh=True) or {}
hold = entry.get("credentialHold")
last_pass = entry.get("pass") if isinstance(entry.get("pass"), dict) else {}
return (isinstance(hold, dict) and hold.get("fingerprint") == fingerprint
and last_pass.get("outcome") == "failed")
def record_resync_state(rt, provider, state, *, connection_id=None):
"""Replace the connector's resync state with a JSON-safe copy of `state`."""
try:
snapshot = json.loads(json.dumps(dict(state or {}), default=str))
except (TypeError, ValueError):
snapshot = {}
snapshot["at"] = _utc_stamp()
def change(entry):
entry["resync"] = snapshot
return _status_update(rt, provider, connection_id, change)
def update_status(rt, provider, section, fields, *, connection_id=None):
"""Merge `fields` into one named section of a connector's entry (first loads, heartbeats)."""
name = _key(section)
if not name or name in _STATUS_RESERVED:
raise ValueError(f"section {section!r} is not writable through update_status")
try:
values = json.loads(json.dumps(dict(fields or {}), default=str))
except (TypeError, ValueError):
values = {}
def change(entry):
current = entry.get(name) if isinstance(entry.get(name), dict) else {}
entry[name] = {**current, **values}
return _status_update(rt, provider, connection_id, change)
def register_provider(provider, *, label, retry_hint):
"""Register provider metadata; runners remain in the API layer."""
provider_key = _key(provider)
if not provider_key:
raise ValueError("provider is required")
_PROVIDERS[provider_key] = {
"key": provider_key,
"label": str(label or provider_key),
"retryHint": str(retry_hint or ""),
}
def providers():
"""Return a copy of registered provider metadata."""
return {key: dict(value) for key, value in _PROVIDERS.items()}
register_provider("odoo", label="Odoo", retry_hint="Retry the Odoo database sync.")
register_provider(
"amazon_sp",
label="Amazon",
retry_hint="Use Sync now to retry the Amazon pull.",
)
register_provider("meta_ads", label="Meta Ads",
retry_hint="Use Sync now to retry the Meta Ads pull.")
register_provider("amazon_ads", label="Amazon Ads",
retry_hint="Use Sync now to retry the Amazon Ads pull.")
"""Tenant-and-connection scoped connector sync lifecycle state.
This data-layer module records what a provider runner did. It deliberately does not import or
call providers: the API layer owns runners while every connector shares this durable vocabulary.
"""
from __future__ import annotations
import datetime as _dt
import hashlib
import json
import logging
import threading
import time
import uuid
from copy import deepcopy
import core.store as store
STATE_KEY = "connector_sync_state"
STATES = ("queued", "syncing", "materializing", "ready", "partial", "error")
_ERROR_LIMIT = 50
_DEFAULT_CONNECTION = "default"
_PROVIDERS = {}
# Ordinary requests coalesce per provider/tenant. Explicit events retain separate durable receipts.
_SYNC_REQUESTS = {}
_SYNC_REQUEST_RUNTIMES = {}
_SYNC_NOT_BEFORE = {}
_SYNC_REQUEST_LOCK = threading.Lock()
_SYNC_WAKE = threading.Event()
_SYNC_DUE_TIMER = None
SYNC_REQUESTS_KEY = "connector_sync_requests_v1"
NOTIFICATION_TARGETING_CONTRACT = "amazon-table-events-v1"
# Ordinary requests the web process hands to the sync worker live in this member of the desk.
REMOTE_MEMBER = "ordinary"
REMOTE_KEEP_SECONDS = 3600
_REQUEST_LISTENERS = []
_REMOTE_IDS = {}
_EVENT_DEDUPE_SECONDS = 600
def _target_tables(provider, tenant, rt, tables, event_id):
from core.connector_spec import AMAZON_TABLES
known = {row.source.removeprefix("amzn_") for row in AMAZON_TABLES
if not row.native.startswith("derived:")}
if (provider != "amazon_sp" or rt is None or rt.key != tenant
or not isinstance(tables, list) or not tables
or any(not isinstance(table, str) or table not in known for table in tables)):
raise ValueError("amazon_notification_target_invalid")
if not isinstance(event_id, str) or not event_id.strip() or len(tables) != 1:
raise ValueError("Amazon notifications require one table and an event ID.")
return list(tables)
def _sync_request_document(current):
document = deepcopy(current) if current is not None else {}
if not isinstance(document, dict):
raise ValueError("The stored sync request desk is unavailable.")
for member in ("requests", "events", REMOTE_MEMBER):
document.setdefault(member, {})
if not isinstance(document[member], dict):
raise ValueError("The stored sync request desk is unavailable.")
return document
def _cache_sync_requests(rt, document):
"""Refresh this tenant's durable receipts under the desk lock without touching ordinary work."""
rows = {}
for identity, row in document["requests"].items():
if not isinstance(row, dict) or row.get("id") != identity or row.get("tenant") != rt.key:
raise ValueError("The stored sync receipt does not match its tenant or identity.")
_target_tables(row.get("provider"), rt.key, rt, row.get("tables"), row.get("eventId"))
rows[(row["provider"], rt.key, identity)] = deepcopy(row)
for key in list(_SYNC_REQUESTS):
if len(key) == 3 and key[1] == rt.key:
del _SYNC_REQUESTS[key]
_SYNC_REQUESTS.update(rows)
_SYNC_REQUEST_RUNTIMES[rt.key] = rt
def _wake_sync_requests(stamp, held=()):
_arm_sync_wake(stamp)
if any(row["state"] == "queued" and (row.get("notBefore") or 0) <= stamp
and row.get("requestedBy") not in held
for row in _SYNC_REQUESTS.values()):
_SYNC_WAKE.set()
def resume_held_requests(now=None):
"""Wake the resident loop for queued work a hold left waiting."""
with _SYNC_REQUEST_LOCK:
_wake_sync_requests(time.time() if now is None else now)
def restore_sync_requests(rt, *, now=None, requeue=True):
"""Restore durable events and requeue claims abandoned by a previous process.
requeue=False only reads: a process that never claims must not requeue another one's claim.
"""