Deploy AIOS web (React glide grid + FastAPI slice)
Browse files- RELEASES.json +1 -1
- VERSION +1 -1
- api/main.py +16 -3
- api/sync_alerts.py +42 -9
- platform/core/connector_sync.py +40 -0
- web/public/AIOS_DEPLOY_RECEIPT +1 -1
- web/src/connectors/ConnectorsPage.tsx +16 -1
RELEASES.json
CHANGED
|
@@ -1,4 +1,4 @@
|
|
| 1 |
{
|
| 2 |
-
"current": "
|
| 3 |
"releases": []
|
| 4 |
}
|
|
|
|
| 1 |
{
|
| 2 |
+
"current": "d99dd1ebf",
|
| 3 |
"releases": []
|
| 4 |
}
|
VERSION
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
|
|
|
|
| 1 |
+
d99dd1ebf
|
api/main.py
CHANGED
|
@@ -1188,16 +1188,29 @@ def _pull_meta(why, tenant_key=None):
|
|
| 1188 |
for key, rt in _meta_scan.live_tenants(runtime, tenants=tenants, name="meta-pull",
|
| 1189 |
on_error=runtime_failed):
|
| 1190 |
try:
|
| 1191 |
-
|
|
|
|
| 1192 |
continue
|
| 1193 |
from core import connector_sync as _meta_status
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1194 |
_meta_status.record_pass_start(rt, "meta_ads")
|
| 1195 |
_status = globals().get("_publish_connector_status") or (lambda *_a, **_k: False)
|
| 1196 |
rep = mirror_store.tracked_sync("meta", rt, _meta.sync, key,
|
| 1197 |
log=lambda *_a: None, insights_days=7)
|
| 1198 |
outcomes[key] = mirror_store._sync_result("meta", rep)[1]
|
| 1199 |
-
detail =
|
|
|
|
| 1200 |
_status(rt, "meta_ads", "done" if outcomes[key] else "failed", detail=detail)
|
|
|
|
|
|
|
| 1201 |
for problem in rep.get("problems") or []:
|
| 1202 |
print(f"[aios-api] meta sync PROBLEM ({why}, {key}): {_sync_now_failure(problem)}")
|
| 1203 |
except Exception as exc:
|
|
@@ -1625,7 +1638,7 @@ def _sync_now_cause(cause):
|
|
| 1625 |
"The provider rate limit stopped the sync. Retry after the provider delay."),
|
| 1626 |
("timeout", ("timed out", "timeout"), "The sync failed because the provider request timed out."),
|
| 1627 |
("credential_refused", ("permission", "access denied", "unauthorized", "invalid oauth", "invalid token",
|
| 1628 |
-
"expired", "authentication"), "
|
| 1629 |
("mirror_constraint", ("duplicate key", "primary key", "constraint"),
|
| 1630 |
"The mirror rejected rows because they violate a database constraint."),
|
| 1631 |
("connect_failed", ("connection", "network", "name resolution"),
|
|
|
|
| 1188 |
for key, rt in _meta_scan.live_tenants(runtime, tenants=tenants, name="meta-pull",
|
| 1189 |
on_error=runtime_failed):
|
| 1190 |
try:
|
| 1191 |
+
connections = _meta._connection_rows(rt, key, {"problems": []})
|
| 1192 |
+
if not connections:
|
| 1193 |
continue
|
| 1194 |
from core import connector_sync as _meta_status
|
| 1195 |
+
# A refused credential is not retried until it changes. An unreadable fingerprint runs the pass.
|
| 1196 |
+
try:
|
| 1197 |
+
fingerprint = _meta_status.credential_fingerprint(
|
| 1198 |
+
[f"{row[0]}\t{row[2]}" for row in connections])
|
| 1199 |
+
except Exception: # noqa: BLE001
|
| 1200 |
+
fingerprint = None
|
| 1201 |
+
if _meta_status.credential_held(rt, "meta_ads", fingerprint):
|
| 1202 |
+
print("[sync] meta_ads held: the provider refused the credential; waiting for a new one")
|
| 1203 |
+
continue
|
| 1204 |
_meta_status.record_pass_start(rt, "meta_ads")
|
| 1205 |
_status = globals().get("_publish_connector_status") or (lambda *_a, **_k: False)
|
| 1206 |
rep = mirror_store.tracked_sync("meta", rt, _meta.sync, key,
|
| 1207 |
log=lambda *_a: None, insights_days=7)
|
| 1208 |
outcomes[key] = mirror_store._sync_result("meta", rep)[1]
|
| 1209 |
+
cause, detail = (_sync_now_cause(" ".join(rep.get("problems") or []))
|
| 1210 |
+
if not outcomes[key] else ("", ""))
|
| 1211 |
_status(rt, "meta_ads", "done" if outcomes[key] else "failed", detail=detail)
|
| 1212 |
+
if cause == "credential_refused":
|
| 1213 |
+
_meta_status.hold_credential(rt, "meta_ads", fingerprint)
|
| 1214 |
for problem in rep.get("problems") or []:
|
| 1215 |
print(f"[aios-api] meta sync PROBLEM ({why}, {key}): {_sync_now_failure(problem)}")
|
| 1216 |
except Exception as exc:
|
|
|
|
| 1638 |
"The provider rate limit stopped the sync. Retry after the provider delay."),
|
| 1639 |
("timeout", ("timed out", "timeout"), "The sync failed because the provider request timed out."),
|
| 1640 |
("credential_refused", ("permission", "access denied", "unauthorized", "invalid oauth", "invalid token",
|
| 1641 |
+
"expired", "authentication"), "Credential expired. Reconnect to resume."),
|
| 1642 |
("mirror_constraint", ("duplicate key", "primary key", "constraint"),
|
| 1643 |
"The mirror rejected rows because they violate a database constraint."),
|
| 1644 |
("connect_failed", ("connection", "network", "name resolution"),
|
api/sync_alerts.py
CHANGED
|
@@ -27,8 +27,18 @@ SYNC_KEY = "sync"
|
|
| 27 |
# Owner 2026-10-29: alert only on real breakage. A report Amazon does not offer this seller never alerts.
|
| 28 |
AMAZON_NOT_OFFERED = "Amazon does not offer this report to this seller account"
|
| 29 |
AMAZON_ROLE = re.compile(r"has not granted access|access to requested resource is denied", re.I)
|
| 30 |
-
# A
|
| 31 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 32 |
# A throttle or a held table alerts only when it repeats on this many passes in a row.
|
| 33 |
REPEAT_CODES = {"throttled": 2, "held": 2}
|
| 34 |
|
|
@@ -120,9 +130,14 @@ def _clean(failure):
|
|
| 120 |
if not isinstance(failure, dict) or not str(failure.get("key") or "").strip():
|
| 121 |
return None
|
| 122 |
key = scrub(failure["key"], 120)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 123 |
return {"key": key, "label": scrub(failure.get("label") or key, 120),
|
| 124 |
-
"code":
|
| 125 |
-
"cause": scrub(failure.get("cause") or "no cause given"),
|
| 126 |
"last_good": failure.get("last_good") if _parse(failure.get("last_good")) else None}
|
| 127 |
|
| 128 |
|
|
@@ -332,6 +347,16 @@ def _item(alert):
|
|
| 332 |
return f"{alert['label']} ({alert['cause']}, {tail})"
|
| 333 |
|
| 334 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 335 |
def environment():
|
| 336 |
"""LIVE, STAGING or LOCAL, from the Space id HF sets, so a staging alert never reads as live."""
|
| 337 |
space = (os.environ.get("SPACE_ID") or os.environ.get("APP_BASE_URL") or "").lower()
|
|
@@ -341,14 +366,22 @@ def environment():
|
|
| 341 |
|
| 342 |
|
| 343 |
def compose(tenant, connector, plan, document):
|
| 344 |
-
"""One line for the whole pass: FAILED, STILL FAILING
|
| 345 |
alerts, recovered = document.get("alerts") or {}, document.get("recovered") or {}
|
| 346 |
where = f"{environment()} {tenant} {connector}"
|
| 347 |
parts = []
|
| 348 |
-
|
| 349 |
-
|
| 350 |
-
|
| 351 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 352 |
names = [recovered[a]["label"] for a in plan["recovered"] if a in recovered]
|
| 353 |
if names:
|
| 354 |
parts.append(f"Loopable sync RECOVERED: {where}: " + ", ".join(names) + ".")
|
|
|
|
| 27 |
# Owner 2026-10-29: alert only on real breakage. A report Amazon does not offer this seller never alerts.
|
| 28 |
AMAZON_NOT_OFFERED = "Amazon does not offer this report to this seller account"
|
| 29 |
AMAZON_ROLE = re.compile(r"has not granted access|access to requested resource is denied", re.I)
|
| 30 |
+
# A refused or expired credential is one cause: the owner reconnects or disconnects the connector.
|
| 31 |
+
CREDENTIAL_CAUSE = "credential expired, reconnect or disconnect it"
|
| 32 |
+
CREDENTIAL_CODES = frozenset({"credential", "credential_refused", "source_auth_failed"})
|
| 33 |
+
# A TLS "certificate has expired" names no token or credential, so it never matches.
|
| 34 |
+
CREDENTIAL = re.compile(
|
| 35 |
+
r"\brefused the connector credential\b|\bcredential expired\b|\binvalid_grant\b|\bcredential_refused\b"
|
| 36 |
+
r"|\berror validating access token\b|\bsession has expired\b"
|
| 37 |
+
r"|\b(?:\w+_)?tokens?\b[^.]{0,40}\b(?:expired|revoked|invalid)\b"
|
| 38 |
+
r"|\b(?:expired|revoked|invalid)\b[^.]{0,20}\b(?:\w+_)?tokens?\b"
|
| 39 |
+
r"|\b(?:refused|rejected)\b[^.]{0,40}\bcredential\b", re.I)
|
| 40 |
+
# A missing role or a refused credential is a standing account fact: it alerts once and never reminds.
|
| 41 |
+
ONCE_CODES = frozenset({"role", "credential"})
|
| 42 |
# A throttle or a held table alerts only when it repeats on this many passes in a row.
|
| 43 |
REPEAT_CODES = {"throttled": 2, "held": 2}
|
| 44 |
|
|
|
|
| 130 |
if not isinstance(failure, dict) or not str(failure.get("key") or "").strip():
|
| 131 |
return None
|
| 132 |
key = scrub(failure["key"], 120)
|
| 133 |
+
code = scrub(failure.get("code") or "error", 60) or "error"
|
| 134 |
+
cause = scrub(failure.get("cause") or "no cause given")
|
| 135 |
+
raw = f"{failure.get('code') or ''} {failure.get('cause') or ''}"
|
| 136 |
+
# A missing Amazon role stays a role: it is checked before the credential.
|
| 137 |
+
if code != "role" and not AMAZON_ROLE.search(raw) and (code in CREDENTIAL_CODES or CREDENTIAL.search(raw)):
|
| 138 |
+
code, cause = "credential", CREDENTIAL_CAUSE
|
| 139 |
return {"key": key, "label": scrub(failure.get("label") or key, 120),
|
| 140 |
+
"code": code, "cause": cause,
|
|
|
|
| 141 |
"last_good": failure.get("last_good") if _parse(failure.get("last_good")) else None}
|
| 142 |
|
| 143 |
|
|
|
|
| 347 |
return f"{alert['label']} ({alert['cause']}, {tail})"
|
| 348 |
|
| 349 |
|
| 350 |
+
#: Connectors whose cards the watchdog reads, so their causes also reach the AI inbox.
|
| 351 |
+
INBOX_CONNECTORS = frozenset({"amazon", "odoo", "meta_ads", "amazon_ads"})
|
| 352 |
+
|
| 353 |
+
|
| 354 |
+
def _names_table(connector, alert):
|
| 355 |
+
"""True when the alert's key is one table: a watchdog `table:` key or an Amazon pass table."""
|
| 356 |
+
key = str(alert.get("key") or "")
|
| 357 |
+
return key.startswith("table:") or (connector == "amazon" and key != SYNC_KEY)
|
| 358 |
+
|
| 359 |
+
|
| 360 |
def environment():
|
| 361 |
"""LIVE, STAGING or LOCAL, from the Space id HF sets, so a staging alert never reads as live."""
|
| 362 |
space = (os.environ.get("SPACE_ID") or os.environ.get("APP_BASE_URL") or "").lower()
|
|
|
|
| 366 |
|
| 367 |
|
| 368 |
def compose(tenant, connector, plan, document):
|
| 369 |
+
"""One line for the whole pass: FAILED lists each alert, STILL FAILING counts them, RECOVERED names them."""
|
| 370 |
alerts, recovered = document.get("alerts") or {}, document.get("recovered") or {}
|
| 371 |
where = f"{environment()} {tenant} {connector}"
|
| 372 |
parts = []
|
| 373 |
+
items = [_item(alerts[a]) for a in sorted(plan["failed"]) if a in alerts]
|
| 374 |
+
if items:
|
| 375 |
+
parts.append(f"Loopable sync FAILED: {where}: " + ", ".join(items) + ".")
|
| 376 |
+
# The daily reminder is one short line per connector: the inbox keeps the full list.
|
| 377 |
+
reminded = [alerts[a] for a in sorted(plan["reminder"]) if a in alerts]
|
| 378 |
+
if reminded:
|
| 379 |
+
tables = all(_names_table(connector, alert) for alert in reminded)
|
| 380 |
+
noun = ("table" if tables else "cause") + ("" if len(reminded) == 1 else "s")
|
| 381 |
+
# Only a connector the watchdog reads has its causes in the AI inbox.
|
| 382 |
+
told = connector.startswith("watchdog ") or connector in INBOX_CONNECTORS
|
| 383 |
+
parts.append(f"Loopable sync STILL FAILING: {where}: {len(reminded)} {noun} still failing"
|
| 384 |
+
+ (", see the inbox." if told else "."))
|
| 385 |
names = [recovered[a]["label"] for a in plan["recovered"] if a in recovered]
|
| 386 |
if names:
|
| 387 |
parts.append(f"Loopable sync RECOVERED: {where}: " + ", ".join(names) + ".")
|
platform/core/connector_sync.py
CHANGED
|
@@ -1110,10 +1110,50 @@ def record_pass_end(rt, provider, outcome, *, at=None, detail="", last_good_sync
|
|
| 1110 |
"minutes": round(duration / 60, 3) if duration is not None else None,
|
| 1111 |
"budgetMinutes": 30, "outcome": outcome}
|
| 1112 |
entry["dailyPasses"] = days
|
|
|
|
|
|
|
|
|
|
| 1113 |
_advance_good_sync(entry, last_good_sync_at)
|
| 1114 |
return _status_update(rt, provider, connection_id, change)
|
| 1115 |
|
| 1116 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1117 |
def record_resync_state(rt, provider, state, *, connection_id=None):
|
| 1118 |
"""Replace the connector's resync state with a JSON-safe copy of `state`."""
|
| 1119 |
try:
|
|
|
|
| 1110 |
"minutes": round(duration / 60, 3) if duration is not None else None,
|
| 1111 |
"budgetMinutes": 30, "outcome": outcome}
|
| 1112 |
entry["dailyPasses"] = days
|
| 1113 |
+
if outcome != "failed":
|
| 1114 |
+
# Any pass that did not fail proves the credential works again.
|
| 1115 |
+
entry.pop("credentialHold", None)
|
| 1116 |
_advance_good_sync(entry, last_good_sync_at)
|
| 1117 |
return _status_update(rt, provider, connection_id, change)
|
| 1118 |
|
| 1119 |
|
| 1120 |
+
def credential_fingerprint(material):
|
| 1121 |
+
"""The first 16 hex of sha256 over the credential material, or None when there is none.
|
| 1122 |
+
|
| 1123 |
+
A list or tuple is joined line by line. The secret itself is never stored or returned.
|
| 1124 |
+
"""
|
| 1125 |
+
if isinstance(material, (list, tuple)):
|
| 1126 |
+
material = "\n".join(str(part) for part in material)
|
| 1127 |
+
if not isinstance(material, str) or not material:
|
| 1128 |
+
return None
|
| 1129 |
+
return hashlib.sha256(material.encode("utf-8")).hexdigest()[:16]
|
| 1130 |
+
|
| 1131 |
+
|
| 1132 |
+
def hold_credential(rt, provider, fingerprint, *, connection_id=None):
|
| 1133 |
+
"""Record that the provider refused the credential with this fingerprint."""
|
| 1134 |
+
if not fingerprint:
|
| 1135 |
+
return False
|
| 1136 |
+
at = _utc_stamp()
|
| 1137 |
+
|
| 1138 |
+
def change(entry):
|
| 1139 |
+
entry["credentialHold"] = {"fingerprint": str(fingerprint), "at": at}
|
| 1140 |
+
return _status_update(rt, provider, connection_id, change)
|
| 1141 |
+
|
| 1142 |
+
|
| 1143 |
+
def credential_held(rt, provider, fingerprint, *, connection_id=None):
|
| 1144 |
+
"""True when a scheduled pass must skip: the last pass failed on this same refused credential.
|
| 1145 |
+
|
| 1146 |
+
An unknown fingerprint or an unreadable record answers False, so the pass runs.
|
| 1147 |
+
"""
|
| 1148 |
+
if not fingerprint:
|
| 1149 |
+
return False
|
| 1150 |
+
entry = status(rt, provider, connection_id, fresh=True) or {}
|
| 1151 |
+
hold = entry.get("credentialHold")
|
| 1152 |
+
last_pass = entry.get("pass") if isinstance(entry.get("pass"), dict) else {}
|
| 1153 |
+
return (isinstance(hold, dict) and hold.get("fingerprint") == fingerprint
|
| 1154 |
+
and last_pass.get("outcome") == "failed")
|
| 1155 |
+
|
| 1156 |
+
|
| 1157 |
def record_resync_state(rt, provider, state, *, connection_id=None):
|
| 1158 |
"""Replace the connector's resync state with a JSON-safe copy of `state`."""
|
| 1159 |
try:
|
web/public/AIOS_DEPLOY_RECEIPT
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
|
|
|
|
| 1 |
+
d45be3ce57b731dad675e4eb6522d2d95b3c51cb9c4eb86cf893b258988b3ace
|
web/src/connectors/ConnectorsPage.tsx
CHANGED
|
@@ -437,6 +437,17 @@ export function staleSyncWarnings(name: string, body: unknown): string[] {
|
|
| 437 |
return [`${name} data is stale: last synced ${syncDate(lastCompletedAt)}`];
|
| 438 |
}
|
| 439 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 440 |
/** One read of a daily source's sync route. A failed read is no warning, never a thrown error. */
|
| 441 |
async function dailySyncWarnings(connectorKey: string, signal: AbortSignal): Promise<string[]> {
|
| 442 |
try {
|
|
@@ -445,6 +456,8 @@ async function dailySyncWarnings(connectorKey: string, signal: AbortSignal): Pro
|
|
| 445 |
});
|
| 446 |
if (!res.ok) return [];
|
| 447 |
const body: unknown = await res.json().catch(() => null);
|
|
|
|
|
|
|
| 448 |
return staleSyncWarnings(DAILY_SOURCE_NAMES[connectorKey] ?? connectorKey, body);
|
| 449 |
} catch {
|
| 450 |
return [];
|
|
@@ -1575,7 +1588,9 @@ export function ConnectorCard({
|
|
| 1575 |
const open = onOpen ?? onPullReviews ?? (() => onKeychain(row.key));
|
| 1576 |
// D-2679: databases that failed alone are named; a stale connector still reads "Stale".
|
| 1577 |
const databasesStale = failedDatabasesStatus(syncWarnings);
|
| 1578 |
-
const status =
|
|
|
|
|
|
|
| 1579 |
<span className="conn-card-status conn-catch-up" role="status">
|
| 1580 |
{catchUp?.active ? catchUpLabel(catchUp) : catchUpError
|
| 1581 |
|| (catchUp?.behind && !catchUpLoading ? behindLabel(catchUp.behind) : "Checking sync")}
|
|
|
|
| 437 |
return [`${name} data is stale: last synced ${syncDate(lastCompletedAt)}`];
|
| 438 |
}
|
| 439 |
|
| 440 |
+
/** The server's sentence when the provider refused the connector credential. */
|
| 441 |
+
export const CREDENTIAL_EXPIRED = "Credential expired. Reconnect to resume.";
|
| 442 |
+
|
| 443 |
+
/** The card line for a failed pass on a refused credential, PURE: none for any other state. */
|
| 444 |
+
export function credentialSyncWarnings(body: unknown): string[] {
|
| 445 |
+
if (!body || typeof body !== "object") return [];
|
| 446 |
+
const { state, detail } = body as { state?: unknown; detail?: unknown };
|
| 447 |
+
return state === "error" && typeof detail === "string" && detail.startsWith(CREDENTIAL_EXPIRED)
|
| 448 |
+
? [CREDENTIAL_EXPIRED] : [];
|
| 449 |
+
}
|
| 450 |
+
|
| 451 |
/** One read of a daily source's sync route. A failed read is no warning, never a thrown error. */
|
| 452 |
async function dailySyncWarnings(connectorKey: string, signal: AbortSignal): Promise<string[]> {
|
| 453 |
try {
|
|
|
|
| 456 |
});
|
| 457 |
if (!res.ok) return [];
|
| 458 |
const body: unknown = await res.json().catch(() => null);
|
| 459 |
+
const credential = credentialSyncWarnings(body);
|
| 460 |
+
if (credential.length) return credential;
|
| 461 |
return staleSyncWarnings(DAILY_SOURCE_NAMES[connectorKey] ?? connectorKey, body);
|
| 462 |
} catch {
|
| 463 |
return [];
|
|
|
|
| 1588 |
const open = onOpen ?? onPullReviews ?? (() => onKeychain(row.key));
|
| 1589 |
// D-2679: databases that failed alone are named; a stale connector still reads "Stale".
|
| 1590 |
const databasesStale = failedDatabasesStatus(syncWarnings);
|
| 1591 |
+
const status = syncWarnings?.includes(CREDENTIAL_EXPIRED) ? (
|
| 1592 |
+
<span className="conn-card-status" role="status">{CREDENTIAL_EXPIRED}</span>
|
| 1593 |
+
) : catchUp?.active || catchUp?.behind || catchUpError || catchUpLoading ? (
|
| 1594 |
<span className="conn-card-status conn-catch-up" role="status">
|
| 1595 |
{catchUp?.active ? catchUpLabel(catchUp) : catchUpError
|
| 1596 |
|| (catchUp?.behind && !catchUpLoading ? behindLabel(catchUp.behind) : "Checking sync")}
|