Spaces:
Running on CPU Upgrade
Running on CPU Upgrade
Deploy AIOS web (React glide grid + FastAPI slice)
Browse files- RELEASES.json +1 -1
- VERSION +1 -1
- api/main.py +24 -1
- api/odoo_relational.py +104 -18
- api/routes_connectors.py +1 -0
- api/sync_alerts.py +372 -0
- web/public/AIOS_DEPLOY_RECEIPT +1 -1
- web/src/connectors/OdooConfig.tsx +4 -2
RELEASES.json
CHANGED
|
@@ -1,5 +1,5 @@
|
|
| 1 |
{
|
| 2 |
-
"current": "
|
| 3 |
"releases": [
|
| 4 |
{
|
| 5 |
"version": "v86",
|
|
|
|
| 1 |
{
|
| 2 |
+
"current": "eda4a2ee5cf5 (eda4a2ee5cf5)",
|
| 3 |
"releases": [
|
| 4 |
{
|
| 5 |
"version": "v86",
|
VERSION
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
|
|
|
|
| 1 |
+
eda4a2ee5cf5 (eda4a2ee5cf5)
|
api/main.py
CHANGED
|
@@ -1111,6 +1111,13 @@ def _pull_amazon(why, vendor=True):
|
|
| 1111 |
print(f"[aios-api] amazon sync done ({why}): " + ", ".join(
|
| 1112 |
f"{table}={row.get('state', 'done')}" + (f", {row['rows']} rows" if 'rows' in row else "")
|
| 1113 |
for table, row in rep["tables"].items()))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1114 |
return not bool(rep.get("problems"))
|
| 1115 |
except Exception as exc: # noqa: BLE001
|
| 1116 |
print(f"[aios-api] amazon sync FAILED ({why}): {type(exc).__name__}: {exc}")
|
|
@@ -2070,13 +2077,22 @@ def _rebuild_odoo_relational(why, previous_meta_fingerprint=None, meta_state_out
|
|
| 2070 |
# one this mirror serves. Best effort: a stamp must never fail the rebuild it records.
|
| 2071 |
if cid == str(default_cid):
|
| 2072 |
# D-2679: a good sync still names the databases that failed by themselves.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2073 |
failed_databases = None if safe_errors else [
|
| 2074 |
{"key": key, "code": db_failure.get("code"),
|
| 2075 |
"stage": db_failure.get("stage"), "at": refreshed_at,
|
| 2076 |
"message": routes_connectors._DATABASE_SAFE_ERRORS.get(
|
| 2077 |
str(db_failure.get("code") or ""))
|
| 2078 |
or routes_connectors._odoo_failure(
|
| 2079 |
-
None, str(db_failure.get("stage") or "materialization"))["message"]
|
|
|
|
| 2080 |
for key, db_failure in blocking.items() if isinstance(db_failure, dict)]
|
| 2081 |
if failed_databases is not None and table_keys is not None:
|
| 2082 |
# D-2679: a changed-tables pass keeps the failures of databases it did not run.
|
|
@@ -2092,6 +2108,13 @@ def _rebuild_odoo_relational(why, previous_meta_fingerprint=None, meta_state_out
|
|
| 2092 |
failed_databases=failed_databases)
|
| 2093 |
except Exception: # noqa: BLE001
|
| 2094 |
pass
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2095 |
meta_state = _rebuild_meta_relational(
|
| 2096 |
why, _rt, previous_fingerprint=previous_meta_fingerprint, force_relations=True)
|
| 2097 |
if isinstance(meta_state_out, dict):
|
|
|
|
| 1111 |
print(f"[aios-api] amazon sync done ({why}): " + ", ".join(
|
| 1112 |
f"{table}={row.get('state', 'done')}" + (f", {row['rows']} rows" if 'rows' in row else "")
|
| 1113 |
for table, row in rep["tables"].items()))
|
| 1114 |
+
# Owner 2026-10-29: an Amazon table that fails or recovers is Telegrammed at once.
|
| 1115 |
+
try:
|
| 1116 |
+
import sync_alerts as _sa
|
| 1117 |
+
_sa.notify(tenant_key, "amazon", _sa.amazon_failures(rep), observed=_sa.amazon_observed(rep),
|
| 1118 |
+
store=rt)
|
| 1119 |
+
except Exception: # noqa: BLE001
|
| 1120 |
+
pass
|
| 1121 |
return not bool(rep.get("problems"))
|
| 1122 |
except Exception as exc: # noqa: BLE001
|
| 1123 |
print(f"[aios-api] amazon sync FAILED ({why}): {type(exc).__name__}: {exc}")
|
|
|
|
| 2077 |
# one this mirror serves. Best effort: a stamp must never fail the rebuild it records.
|
| 2078 |
if cid == str(default_cid):
|
| 2079 |
# D-2679: a good sync still names the databases that failed by themselves.
|
| 2080 |
+
# The health cleaner drops `cause`, so it is read from the raw report entry.
|
| 2081 |
+
def _raw_cause(key):
|
| 2082 |
+
receipt = all_receipts.get(key) if isinstance(all_receipts.get(key), dict) else {}
|
| 2083 |
+
for raw in ((report.get("databaseErrors") or {}).get(key),
|
| 2084 |
+
receipt.get("error"), receipt.get("sourceError")):
|
| 2085 |
+
if isinstance(raw, dict) and raw.get("cause"):
|
| 2086 |
+
return str(raw["cause"])
|
| 2087 |
+
return None
|
| 2088 |
failed_databases = None if safe_errors else [
|
| 2089 |
{"key": key, "code": db_failure.get("code"),
|
| 2090 |
"stage": db_failure.get("stage"), "at": refreshed_at,
|
| 2091 |
"message": routes_connectors._DATABASE_SAFE_ERRORS.get(
|
| 2092 |
str(db_failure.get("code") or ""))
|
| 2093 |
or routes_connectors._odoo_failure(
|
| 2094 |
+
None, str(db_failure.get("stage") or "materialization"))["message"],
|
| 2095 |
+
**({"cause": _raw_cause(key)} if _raw_cause(key) else {})}
|
| 2096 |
for key, db_failure in blocking.items() if isinstance(db_failure, dict)]
|
| 2097 |
if failed_databases is not None and table_keys is not None:
|
| 2098 |
# D-2679: a changed-tables pass keeps the failures of databases it did not run.
|
|
|
|
| 2108 |
failed_databases=failed_databases)
|
| 2109 |
except Exception: # noqa: BLE001
|
| 2110 |
pass
|
| 2111 |
+
# Owner 2026-10-29: a database that fails or recovers is Telegrammed at once.
|
| 2112 |
+
try:
|
| 2113 |
+
import sync_alerts as _sa
|
| 2114 |
+
_sa.notify(tenant_key, "odoo", _sa.odoo_failures(failed_databases, safe_errors),
|
| 2115 |
+
observed=_sa.odoo_observed(failed_databases), store=_rt)
|
| 2116 |
+
except Exception: # noqa: BLE001
|
| 2117 |
+
pass
|
| 2118 |
meta_state = _rebuild_meta_relational(
|
| 2119 |
why, _rt, previous_fingerprint=previous_meta_fingerprint, force_relations=True)
|
| 2120 |
if isinstance(meta_state_out, dict):
|
api/odoo_relational.py
CHANGED
|
@@ -3951,6 +3951,14 @@ def capture_source_counts(rt, connection_id=None, buckets=None):
|
|
| 3951 |
counts[bucket] = _source_domain_count(client, bucket)
|
| 3952 |
except Exception: # noqa: BLE001
|
| 3953 |
continue
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3954 |
return counts
|
| 3955 |
|
| 3956 |
|
|
@@ -5700,7 +5708,8 @@ def record_sync_outcome(rt, ok, at=None, failed_databases=None):
|
|
| 5700 |
"message": _plain(item.get("message")
|
| 5701 |
or "The database could not be materialized.", 300),
|
| 5702 |
"stage": _plain(item.get("stage") or "materialization", 40),
|
| 5703 |
-
"at": str(item.get("at") or stamp)
|
|
|
|
| 5704 |
|
| 5705 |
def _up(cur):
|
| 5706 |
cur = dict(cur) if isinstance(cur, dict) else {}
|
|
@@ -5925,7 +5934,8 @@ def sync_freshness(rt, now=None):
|
|
| 5925 |
failed_databases.append({
|
| 5926 |
name: (" ".join(str(item.get(name) or "").replace("—", ",")
|
| 5927 |
.replace("–", ",").split())[:300] or None)
|
| 5928 |
-
for name in ("key", "label", "code", "message", "stage", "at")
|
|
|
|
| 5929 |
return {"stale": bool(stale), "lastGoodSyncAt": last_good,
|
| 5930 |
"failedDatabases": failed_databases,
|
| 5931 |
"schedule": cfg["syncEvery"], "staleAfterSeconds": allowed,
|
|
@@ -6182,8 +6192,25 @@ def read_shipments_live(client):
|
|
| 6182 |
return rows
|
| 6183 |
|
| 6184 |
|
| 6185 |
-
def
|
| 6186 |
-
"""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6187 |
year = int(year)
|
| 6188 |
start, end = f"{year:04d}-01-01", f"{year + 1:04d}-01-01"
|
| 6189 |
if hasattr(rows, "execute"):
|
|
@@ -6197,11 +6224,14 @@ def reconcile_shipments(rows, client, year):
|
|
| 6197 |
values = rows.values() if isinstance(rows, dict) else rows
|
| 6198 |
count = sum(1 for row in values if row.get("state") == "done"
|
| 6199 |
and start <= (row.get("date_done") or "") < end)
|
| 6200 |
-
|
| 6201 |
-
|
| 6202 |
-
|
| 6203 |
-
|
| 6204 |
-
|
|
|
|
|
|
|
|
|
|
| 6205 |
|
| 6206 |
|
| 6207 |
def shipment_source_spec():
|
|
@@ -6310,8 +6340,11 @@ def apply_shipments(rt, planned, username="automation", today=None):
|
|
| 6310 |
|
| 6311 |
|
| 6312 |
def refresh_shipments(rt, *, connection_id=None, year=None, today=None,
|
| 6313 |
-
username="automation", client=None, cur=None):
|
| 6314 |
-
"""Reconcile the tenant's shipment mirror against an independent Odoo count.
|
|
|
|
|
|
|
|
|
|
| 6315 |
if client is None:
|
| 6316 |
source = rt.odoo_source() if connection_id is None else rt.odoo_source(connection_id)
|
| 6317 |
if source is None:
|
|
@@ -6319,11 +6352,12 @@ def refresh_shipments(rt, *, connection_id=None, year=None, today=None,
|
|
| 6319 |
from core.odoo import get_odoo
|
| 6320 |
with source._bound():
|
| 6321 |
return refresh_shipments(rt, connection_id=connection_id, year=year, today=today,
|
| 6322 |
-
username=username, client=get_odoo(), cur=cur
|
|
|
|
| 6323 |
stamp = today or _iso_today()
|
| 6324 |
year = int(year) if year is not None else int(stamp[:4])
|
| 6325 |
planned = plan_shipments(rt, connection_id, client, stamp, cur)
|
| 6326 |
-
reconciled = reconcile_shipments(planned["rows"], client, year)
|
| 6327 |
if not reconciled["ok"]:
|
| 6328 |
raise Refused(f"stock.picking: {year} done outgoing shipments do not reconcile "
|
| 6329 |
f"({reconciled['rowCount']} mirror, {reconciled['sourceCount']} source)")
|
|
@@ -7032,6 +7066,48 @@ def health_view(report):
|
|
| 7032 |
return view
|
| 7033 |
|
| 7034 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 7035 |
def refresh_curated(rt, cur, buckets, *, connection_id=None, username="automation",
|
| 7036 |
today=None, retire=True, on_apply=None, source_client=None,
|
| 7037 |
source_counts=None):
|
|
@@ -7062,7 +7138,9 @@ def refresh_curated(rt, cur, buckets, *, connection_id=None, username="automatio
|
|
| 7062 |
source_failure = {
|
| 7063 |
"code": "source_reconciliation_failed", "type": type(exc).__name__,
|
| 7064 |
"message": "This Odoo database source population could not be counted.",
|
|
|
|
| 7065 |
}
|
|
|
|
| 7066 |
identity = {
|
| 7067 |
"databaseKey": key,
|
| 7068 |
"schemaKey": key,
|
|
@@ -7130,10 +7208,12 @@ def refresh_curated(rt, cur, buckets, *, connection_id=None, username="automatio
|
|
| 7130 |
if source_failure:
|
| 7131 |
errors[key] = source_failure
|
| 7132 |
except Exception as exc:
|
| 7133 |
-
# Exception prose may contain connection or record values
|
| 7134 |
-
# structural class and a fixed product message
|
| 7135 |
failure = {"code": "materialization_failed", "type": type(exc).__name__,
|
| 7136 |
-
"message": "This Odoo database could not be materialized."
|
|
|
|
|
|
|
| 7137 |
errors[key] = failure
|
| 7138 |
receipts[key] = {
|
| 7139 |
**identity, "state": "failed", "sourceCount": source_count,
|
|
@@ -7296,16 +7376,22 @@ def refresh(rt, tenant, username="automation", cur=None, today=None, connection_
|
|
| 7296 |
key = table_key_for(SHIPMENTS_KEY, ns)
|
| 7297 |
year = int((today or _iso_today())[:4])
|
| 7298 |
receipt = database_receipts.setdefault(key, {"databaseKey": key})
|
|
|
|
|
|
|
|
|
|
| 7299 |
try:
|
| 7300 |
shipment_report = refresh_shipments(
|
| 7301 |
rt, connection_id=connection_id, year=year, today=today,
|
| 7302 |
-
username=username, cur=cur
|
|
|
|
| 7303 |
receipt["shipmentReconciliation"] = shipment_report["reconcile_mirror"]
|
| 7304 |
except Exception as exc: # noqa: BLE001
|
| 7305 |
# A source refusal belongs to this database and must not abort the pass.
|
| 7306 |
failure = {"code": "source_reconciliation_failed", "type": type(exc).__name__,
|
| 7307 |
"message": f"Completed outgoing shipments could not be reconciled "
|
| 7308 |
-
f"with Odoo for {year}."
|
|
|
|
|
|
|
| 7309 |
database_errors.setdefault(key, failure)
|
| 7310 |
receipt["shipmentReconciliation"] = {"year": year, "ok": False}
|
| 7311 |
receipt["sourceError"] = failure
|
|
|
|
| 3951 |
counts[bucket] = _source_domain_count(client, bucket)
|
| 3952 |
except Exception: # noqa: BLE001
|
| 3953 |
continue
|
| 3954 |
+
if "shipments" in chosen:
|
| 3955 |
+
# The year's done count reconciles ut_odoo_shipments against the pull, not a later Odoo.
|
| 3956 |
+
year = int(_iso_today()[:4])
|
| 3957 |
+
try:
|
| 3958 |
+
counts[shipments_done_count_key(year)] = (
|
| 3959 |
+
int(_shipments_done_source_count(client, year)), "search_count")
|
| 3960 |
+
except Exception: # noqa: BLE001
|
| 3961 |
+
pass
|
| 3962 |
return counts
|
| 3963 |
|
| 3964 |
|
|
|
|
| 5708 |
"message": _plain(item.get("message")
|
| 5709 |
or "The database could not be materialized.", 300),
|
| 5710 |
"stage": _plain(item.get("stage") or "materialization", 40),
|
| 5711 |
+
"at": str(item.get("at") or stamp),
|
| 5712 |
+
**({"cause": _plain(item["cause"], 300)} if item.get("cause") else {})})
|
| 5713 |
|
| 5714 |
def _up(cur):
|
| 5715 |
cur = dict(cur) if isinstance(cur, dict) else {}
|
|
|
|
| 5934 |
failed_databases.append({
|
| 5935 |
name: (" ".join(str(item.get(name) or "").replace("—", ",")
|
| 5936 |
.replace("–", ",").split())[:300] or None)
|
| 5937 |
+
for name in ("key", "label", "code", "message", "stage", "at")
|
| 5938 |
+
+ (("cause",) if item.get("cause") else ())})
|
| 5939 |
return {"stale": bool(stale), "lastGoodSyncAt": last_good,
|
| 5940 |
"failedDatabases": failed_databases,
|
| 5941 |
"schedule": cfg["syncEvery"], "staleAfterSeconds": allowed,
|
|
|
|
| 6192 |
return rows
|
| 6193 |
|
| 6194 |
|
| 6195 |
+
def shipments_done_count_key(year):
|
| 6196 |
+
"""The `capture_source_counts` key for one year's completed outgoing shipment count."""
|
| 6197 |
+
return f"shipments_done_{int(year):04d}"
|
| 6198 |
+
|
| 6199 |
+
|
| 6200 |
+
def _shipments_done_source_count(client, year):
|
| 6201 |
+
"""Odoo's count of completed outgoing shipments in one naive UTC calendar year."""
|
| 6202 |
+
year = int(year)
|
| 6203 |
+
start, end = f"{year:04d}-01-01", f"{year + 1:04d}-01-01"
|
| 6204 |
+
return client.search_count("stock.picking", [
|
| 6205 |
+
("picking_type_code", "=", "outgoing"), ("state", "=", "done"),
|
| 6206 |
+
("date_done", ">=", start + " 00:00:00"), ("date_done", "<", end + " 00:00:00")])
|
| 6207 |
+
|
| 6208 |
+
|
| 6209 |
+
def reconcile_shipments(rows, client, year, source_count=None):
|
| 6210 |
+
"""Independent source count for completed outgoing shipments in one UTC calendar year.
|
| 6211 |
+
|
| 6212 |
+
`source_count` is the count taken at mirror-pull time. Without it Odoo is counted now.
|
| 6213 |
+
"""
|
| 6214 |
year = int(year)
|
| 6215 |
start, end = f"{year:04d}-01-01", f"{year + 1:04d}-01-01"
|
| 6216 |
if hasattr(rows, "execute"):
|
|
|
|
| 6224 |
values = rows.values() if isinstance(rows, dict) else rows
|
| 6225 |
count = sum(1 for row in values if row.get("state") == "done"
|
| 6226 |
and start <= (row.get("date_done") or "") < end)
|
| 6227 |
+
captured = source_count is not None
|
| 6228 |
+
if not captured:
|
| 6229 |
+
source_count = _shipments_done_source_count(client, year)
|
| 6230 |
+
result = {"year": year, "rowCount": count, "sourceCount": source_count,
|
| 6231 |
+
"ok": type(source_count) is int and source_count == count}
|
| 6232 |
+
if captured:
|
| 6233 |
+
result["sourceCountMode"] = "pull_time"
|
| 6234 |
+
return result
|
| 6235 |
|
| 6236 |
|
| 6237 |
def shipment_source_spec():
|
|
|
|
| 6340 |
|
| 6341 |
|
| 6342 |
def refresh_shipments(rt, *, connection_id=None, year=None, today=None,
|
| 6343 |
+
username="automation", client=None, cur=None, source_count=None):
|
| 6344 |
+
"""Reconcile the tenant's shipment mirror against an independent Odoo count.
|
| 6345 |
+
|
| 6346 |
+
`source_count` is the year's done count taken at mirror-pull time, if one was captured.
|
| 6347 |
+
"""
|
| 6348 |
if client is None:
|
| 6349 |
source = rt.odoo_source() if connection_id is None else rt.odoo_source(connection_id)
|
| 6350 |
if source is None:
|
|
|
|
| 6352 |
from core.odoo import get_odoo
|
| 6353 |
with source._bound():
|
| 6354 |
return refresh_shipments(rt, connection_id=connection_id, year=year, today=today,
|
| 6355 |
+
username=username, client=get_odoo(), cur=cur,
|
| 6356 |
+
source_count=source_count)
|
| 6357 |
stamp = today or _iso_today()
|
| 6358 |
year = int(year) if year is not None else int(stamp[:4])
|
| 6359 |
planned = plan_shipments(rt, connection_id, client, stamp, cur)
|
| 6360 |
+
reconciled = reconcile_shipments(planned["rows"], client, year, source_count=source_count)
|
| 6361 |
if not reconciled["ok"]:
|
| 6362 |
raise Refused(f"stock.picking: {year} done outgoing shipments do not reconcile "
|
| 6363 |
f"({reconciled['rowCount']} mirror, {reconciled['sourceCount']} source)")
|
|
|
|
| 7066 |
return view
|
| 7067 |
|
| 7068 |
|
| 7069 |
+
import re as _re_scrub # noqa: E402 (kept beside the scrubber it serves)
|
| 7070 |
+
|
| 7071 |
+
_SCRUB_RULES = (
|
| 7072 |
+
(_re_scrub.compile(r"\b[a-z][a-z0-9+.\-]*://\S+", _re_scrub.I), "<url>"),
|
| 7073 |
+
(_re_scrub.compile(r"[\w.+\-]+@[\w\-]+(?:\.[\w\-]+)+"), "<email>"),
|
| 7074 |
+
(_re_scrub.compile(r"\bbearer\s+\S+", _re_scrub.I), "Bearer <redacted>"),
|
| 7075 |
+
(_re_scrub.compile(r"\b(password|passwd|pwd|secret|token|api[_\-]?key|apikey|authorization"
|
| 7076 |
+
r"|session[_\-]?id|cookie|access[_\-]?key|client[_\-]?secret)\b"
|
| 7077 |
+
r"([\"']?\s*[:=]\s*)(\"[^\"]*\"|'[^']*'|\S+)", _re_scrub.I),
|
| 7078 |
+
r"\1\2<redacted>"),
|
| 7079 |
+
(_re_scrub.compile(r"\b\d{1,3}(?:\.\d{1,3}){3}(?::\d+)?\b"), "<host>"),
|
| 7080 |
+
(_re_scrub.compile(r"\b(?=[A-Za-z0-9_\-]*\d)(?=[A-Za-z0-9_\-]*[A-Za-z])[A-Za-z0-9_\-]{20,}\b"),
|
| 7081 |
+
"<token>"),
|
| 7082 |
+
(_re_scrub.compile(r"=\([^)]*\)"), "=(<value>)"),
|
| 7083 |
+
(_re_scrub.compile(r"\d{9,}"), "<number>"),
|
| 7084 |
+
)
|
| 7085 |
+
_SCRUB_QUOTED = _re_scrub.compile(r"'([^'\n]*)'|\"([^\"\n]*)\"")
|
| 7086 |
+
_SCRUB_NAME = _re_scrub.compile(r"[A-Za-z_][A-Za-z0-9_.]*")
|
| 7087 |
+
|
| 7088 |
+
|
| 7089 |
+
def _scrub_cause(exc):
|
| 7090 |
+
"""`Type: message` for a log and a card, with credentials and record values masked, 300 chars.
|
| 7091 |
+
|
| 7092 |
+
Names (models, tables, columns) and counts stay. Quoted free text is a record value and goes.
|
| 7093 |
+
"""
|
| 7094 |
+
name = type(exc).__name__ if isinstance(exc, BaseException) else "Error"
|
| 7095 |
+
text = str(exc if exc is not None else "")
|
| 7096 |
+
for pattern, mask in _SCRUB_RULES:
|
| 7097 |
+
text = pattern.sub(mask, text)
|
| 7098 |
+
text = _SCRUB_QUOTED.sub(
|
| 7099 |
+
lambda m: m.group(0) if _SCRUB_NAME.fullmatch(m.group(1) or m.group(2) or "")
|
| 7100 |
+
else "'<value>'", text)
|
| 7101 |
+
text = " ".join(text.replace(chr(0x2014), ",").replace(chr(0x2013), ",").split())
|
| 7102 |
+
return (f"{name}: {text}" if text else name)[:300]
|
| 7103 |
+
|
| 7104 |
+
|
| 7105 |
+
def _log_database_failure(key, failure):
|
| 7106 |
+
"""One log line naming a failed database, its code and its scrubbed cause."""
|
| 7107 |
+
print(f"[aios-api] odoo database failed: {key} {failure.get('code')} {failure.get('cause')}",
|
| 7108 |
+
flush=True)
|
| 7109 |
+
|
| 7110 |
+
|
| 7111 |
def refresh_curated(rt, cur, buckets, *, connection_id=None, username="automation",
|
| 7112 |
today=None, retire=True, on_apply=None, source_client=None,
|
| 7113 |
source_counts=None):
|
|
|
|
| 7138 |
source_failure = {
|
| 7139 |
"code": "source_reconciliation_failed", "type": type(exc).__name__,
|
| 7140 |
"message": "This Odoo database source population could not be counted.",
|
| 7141 |
+
"cause": _scrub_cause(exc),
|
| 7142 |
}
|
| 7143 |
+
_log_database_failure(key, source_failure)
|
| 7144 |
identity = {
|
| 7145 |
"databaseKey": key,
|
| 7146 |
"schemaKey": key,
|
|
|
|
| 7208 |
if source_failure:
|
| 7209 |
errors[key] = source_failure
|
| 7210 |
except Exception as exc:
|
| 7211 |
+
# Exception prose may contain connection or record values: only the scrubbed cause
|
| 7212 |
+
# carries it, beside the structural class and a fixed product message.
|
| 7213 |
failure = {"code": "materialization_failed", "type": type(exc).__name__,
|
| 7214 |
+
"message": "This Odoo database could not be materialized.",
|
| 7215 |
+
"cause": _scrub_cause(exc)}
|
| 7216 |
+
_log_database_failure(key, failure)
|
| 7217 |
errors[key] = failure
|
| 7218 |
receipts[key] = {
|
| 7219 |
**identity, "state": "failed", "sourceCount": source_count,
|
|
|
|
| 7376 |
key = table_key_for(SHIPMENTS_KEY, ns)
|
| 7377 |
year = int((today or _iso_today())[:4])
|
| 7378 |
receipt = database_receipts.setdefault(key, {"databaseKey": key})
|
| 7379 |
+
# Pull-time counts describe the default connection's mirror only.
|
| 7380 |
+
captured = ((source_counts or {}).get(shipments_done_count_key(year))
|
| 7381 |
+
if ns is None else None)
|
| 7382 |
try:
|
| 7383 |
shipment_report = refresh_shipments(
|
| 7384 |
rt, connection_id=connection_id, year=year, today=today,
|
| 7385 |
+
username=username, cur=cur,
|
| 7386 |
+
**({"source_count": captured[0]} if captured else {}))
|
| 7387 |
receipt["shipmentReconciliation"] = shipment_report["reconcile_mirror"]
|
| 7388 |
except Exception as exc: # noqa: BLE001
|
| 7389 |
# A source refusal belongs to this database and must not abort the pass.
|
| 7390 |
failure = {"code": "source_reconciliation_failed", "type": type(exc).__name__,
|
| 7391 |
"message": f"Completed outgoing shipments could not be reconciled "
|
| 7392 |
+
f"with Odoo for {year}.",
|
| 7393 |
+
"cause": _scrub_cause(exc)}
|
| 7394 |
+
_log_database_failure(key, failure)
|
| 7395 |
database_errors.setdefault(key, failure)
|
| 7396 |
receipt["shipmentReconciliation"] = {"year": year, "ok": False}
|
| 7397 |
receipt["sourceError"] = failure
|
api/routes_connectors.py
CHANGED
|
@@ -354,6 +354,7 @@ _DATABASE_SAFE_ERRORS = {
|
|
| 354 |
"mirror_pull_failed": "The Odoo mirror pull failed.",
|
| 355 |
"materialization_failed": "The database could not be materialized.",
|
| 356 |
"reconciliation_failed": "The database reconciliation failed.",
|
|
|
|
| 357 |
"freshness_unconfirmed": "Freshness could not be confirmed for this database.",
|
| 358 |
"sync_status_unavailable": "Sync status is unavailable for this database.",
|
| 359 |
"sync_stalled": "This database sync is stalled.",
|
|
|
|
| 354 |
"mirror_pull_failed": "The Odoo mirror pull failed.",
|
| 355 |
"materialization_failed": "The database could not be materialized.",
|
| 356 |
"reconciliation_failed": "The database reconciliation failed.",
|
| 357 |
+
"source_reconciliation_failed": "Its row count could not be matched to Odoo.",
|
| 358 |
"freshness_unconfirmed": "Freshness could not be confirmed for this database.",
|
| 359 |
"sync_status_unavailable": "Sync status is unavailable for this database.",
|
| 360 |
"sync_stalled": "This database sync is stalled.",
|
api/sync_alerts.py
ADDED
|
@@ -0,0 +1,372 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Tell the owner on Telegram, once, when a connector database fails, recovers or stays failed for a day."""
|
| 2 |
+
from __future__ import annotations
|
| 3 |
+
|
| 4 |
+
from datetime import datetime, timedelta, timezone
|
| 5 |
+
import json
|
| 6 |
+
import os
|
| 7 |
+
from pathlib import Path
|
| 8 |
+
import re
|
| 9 |
+
import sys
|
| 10 |
+
from uuid import uuid4
|
| 11 |
+
|
| 12 |
+
DOCUMENT = "ops_sync_alerts"
|
| 13 |
+
REMIND_AFTER = timedelta(hours=24)
|
| 14 |
+
LEASE = timedelta(minutes=5)
|
| 15 |
+
CAUSE_CAP = 300
|
| 16 |
+
MESSAGE_CAP = 1500
|
| 17 |
+
_OK_CAP = 200
|
| 18 |
+
# Amazon table states that are a failure, and the one state that proves a table healthy.
|
| 19 |
+
# `throttled` is left out: Amazon asked us to wait and the next pass retries it, so it is not a failure.
|
| 20 |
+
AMAZON_FAILED = frozenset({"refused", "failed", "error"})
|
| 21 |
+
AMAZON_GOOD = frozenset({"done"})
|
| 22 |
+
SYNC_KEY = "sync"
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
def utc_now():
|
| 26 |
+
return datetime.now(timezone.utc)
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
def store_backend():
|
| 30 |
+
sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "platform"))
|
| 31 |
+
import core.store as store
|
| 32 |
+
return store
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
def default_sender(text):
|
| 36 |
+
sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "platform"))
|
| 37 |
+
from core.neon_usage import send_telegram
|
| 38 |
+
return send_telegram(text)
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
# ---- text -----------------------------------------------------------------------------------
|
| 42 |
+
|
| 43 |
+
_SCRUB = [
|
| 44 |
+
(re.compile(r"(?i)\b([a-z][a-z0-9+.-]*://)[^/\s:@]+:[^/\s@]+@"), r"\1[credentials]@"),
|
| 45 |
+
(re.compile(r"[\w.+-]+@[\w-]+(?:\.[\w-]+)+"), "[email]"),
|
| 46 |
+
(re.compile(r"(?i)\bbearer\s+[A-Za-z0-9._~+/=-]+"), "Bearer [token]"),
|
| 47 |
+
(re.compile(r"\bAtz[ar]\|[^\s,;\"']+"), "[token]"),
|
| 48 |
+
(re.compile(r"(?i)\b(bot)\d+:[A-Za-z0-9_-]+"), r"\1[token]"),
|
| 49 |
+
(re.compile(r"(?i)((?:^|[?&;\s])(?:access_token|refresh_token|token|key|api_key|apikey|secret|"
|
| 50 |
+
r"client_secret|password|passwd|pwd|signature|sig)=)[^&\s\"']+"), r"\1[masked]"),
|
| 51 |
+
(re.compile(r"(?i)\b((?:access_token|refresh_token|client_secret|password|secret|api_key)"
|
| 52 |
+
r"[\"']?\s*[:=]\s*[\"']?)[^\s,;\"'}]+"), r"\1[masked]"),
|
| 53 |
+
# A long run holding a digit and a letter is a key; plain snake_case table names are not.
|
| 54 |
+
(re.compile(r"\b(?=[\w\-+/=]*\d)(?=[\w\-+/=]*[A-Za-z])[\w\-+/=]{32,}"), "[token]"),
|
| 55 |
+
]
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
def scrub(text, cap=CAUSE_CAP):
|
| 59 |
+
"""Mask credentials, tokens and emails, drop dashes the owner must not read, and cap the length."""
|
| 60 |
+
text = str(text if text is not None else "").replace("\u2014", ", ").replace("\u2013", " to ")
|
| 61 |
+
text = re.sub(r"\s+([,.;:])", r"\1", " ".join(text.split()))
|
| 62 |
+
for pattern, replacement in _SCRUB:
|
| 63 |
+
text = pattern.sub(replacement, text)
|
| 64 |
+
return text if len(text) <= cap else text[:cap - 3].rstrip() + "..."
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
def _parse(iso):
|
| 68 |
+
try:
|
| 69 |
+
value = datetime.fromisoformat(str(iso).replace("Z", "+00:00"))
|
| 70 |
+
except (TypeError, ValueError):
|
| 71 |
+
return None
|
| 72 |
+
return value.replace(tzinfo=timezone.utc) if value.tzinfo is None else value.astimezone(timezone.utc)
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
def _nth_sunday(year, month, n):
|
| 76 |
+
first = datetime(year, month, 1, tzinfo=timezone.utc)
|
| 77 |
+
return first + timedelta(days=(6 - first.weekday()) % 7 + 7 * (n - 1))
|
| 78 |
+
|
| 79 |
+
|
| 80 |
+
def _new_york_offset(moment):
|
| 81 |
+
# EDT from the second Sunday of March 2 AM EST to the first Sunday of November 2 AM EDT.
|
| 82 |
+
start = _nth_sunday(moment.year, 3, 2) + timedelta(hours=7)
|
| 83 |
+
end = _nth_sunday(moment.year, 11, 1) + timedelta(hours=6)
|
| 84 |
+
return timedelta(hours=-4) if start <= moment < end else timedelta(hours=-5)
|
| 85 |
+
|
| 86 |
+
|
| 87 |
+
def _clock(moment):
|
| 88 |
+
hour = moment.hour % 12 or 12
|
| 89 |
+
return f"{moment.day} {moment.strftime('%b')} {hour}:{moment.minute:02d} {'AM' if moment.hour < 12 else 'PM'}"
|
| 90 |
+
|
| 91 |
+
|
| 92 |
+
def when(iso):
|
| 93 |
+
"""One instant as Jakarta and New York time, side by side; never a bare UTC time."""
|
| 94 |
+
moment = _parse(iso)
|
| 95 |
+
if moment is None:
|
| 96 |
+
return "an unknown time"
|
| 97 |
+
jakarta = moment + timedelta(hours=7)
|
| 98 |
+
new_york = moment + _new_york_offset(moment)
|
| 99 |
+
return f"{_clock(jakarta)} Jakarta / {_clock(new_york)} New York"
|
| 100 |
+
|
| 101 |
+
|
| 102 |
+
# ---- failure lists --------------------------------------------------------------------------
|
| 103 |
+
|
| 104 |
+
def _clean(failure):
|
| 105 |
+
if not isinstance(failure, dict) or not str(failure.get("key") or "").strip():
|
| 106 |
+
return None
|
| 107 |
+
key = scrub(failure["key"], 120)
|
| 108 |
+
return {"key": key, "label": scrub(failure.get("label") or key, 120),
|
| 109 |
+
"code": scrub(failure.get("code") or "error", 60) or "error",
|
| 110 |
+
"cause": scrub(failure.get("cause") or "no cause given"),
|
| 111 |
+
"last_good": failure.get("last_good") if _parse(failure.get("last_good")) else None}
|
| 112 |
+
|
| 113 |
+
|
| 114 |
+
def odoo_failures(failed_databases=None, safe_errors=None):
|
| 115 |
+
"""Odoo's per-database failures, or the whole connection's errors when the pass failed outright."""
|
| 116 |
+
out = []
|
| 117 |
+
for item in failed_databases or []:
|
| 118 |
+
if isinstance(item, dict) and item.get("key"):
|
| 119 |
+
out.append({"key": str(item["key"]), "label": str(item.get("label") or item["key"]),
|
| 120 |
+
"code": str(item.get("code") or item.get("stage") or "error"),
|
| 121 |
+
"cause": str(item.get("cause") or item.get("message")
|
| 122 |
+
or "the database did not sync")})
|
| 123 |
+
for text in safe_errors or []:
|
| 124 |
+
code, _, cause = str(text).partition(":")
|
| 125 |
+
out.append({"key": "connection", "label": "Odoo connection", "code": code.strip() or "error",
|
| 126 |
+
"cause": cause.strip() or code.strip()})
|
| 127 |
+
return out
|
| 128 |
+
|
| 129 |
+
|
| 130 |
+
def odoo_observed(failed_databases=None):
|
| 131 |
+
"""A failed connection checked no database, so per-database alerts carry forward; None means a full list."""
|
| 132 |
+
return None if failed_databases is not None else {"connection"}
|
| 133 |
+
|
| 134 |
+
|
| 135 |
+
def amazon_failures(report):
|
| 136 |
+
"""Amazon's refused or failed tables plus its pass-level problems, one entry per table or kind."""
|
| 137 |
+
report = report if isinstance(report, dict) else {}
|
| 138 |
+
tables = report.get("tables") if isinstance(report.get("tables"), dict) else {}
|
| 139 |
+
merged = {}
|
| 140 |
+
|
| 141 |
+
def add(key, label, code, cause):
|
| 142 |
+
entry = merged.setdefault((key, code), {"key": key, "label": label, "code": code, "causes": []})
|
| 143 |
+
if cause and cause not in entry["causes"]:
|
| 144 |
+
entry["causes"].append(cause)
|
| 145 |
+
|
| 146 |
+
covered = set()
|
| 147 |
+
for table, row in tables.items():
|
| 148 |
+
row = row if isinstance(row, dict) else {}
|
| 149 |
+
state = str(row.get("state") or "")
|
| 150 |
+
if state in AMAZON_FAILED:
|
| 151 |
+
covered.add(str(table))
|
| 152 |
+
# `reason` is the plain sentence; `diagnostic` carries raw exception text.
|
| 153 |
+
add(str(table), str(table), state, str(row.get("reason") or state))
|
| 154 |
+
for problem in report.get("problems") or []:
|
| 155 |
+
text = str(problem)
|
| 156 |
+
head = re.match(r"([A-Za-z0-9_]+):\s", text)
|
| 157 |
+
name = head.group(1) if head else None
|
| 158 |
+
if name in covered:
|
| 159 |
+
continue
|
| 160 |
+
if name in tables:
|
| 161 |
+
add(name, name, "problem", text[head.end():])
|
| 162 |
+
else:
|
| 163 |
+
add(SYNC_KEY, "Amazon sync", "problem", text)
|
| 164 |
+
return [{"key": e["key"], "label": e["label"], "code": e["code"],
|
| 165 |
+
"cause": scrub(" / ".join(e["causes"]) or e["code"])} for e in merged.values()]
|
| 166 |
+
|
| 167 |
+
|
| 168 |
+
def amazon_observed(report):
|
| 169 |
+
"""The keys this Amazon pass proved healthy or failing; any other key keeps its alert."""
|
| 170 |
+
report = report if isinstance(report, dict) else {}
|
| 171 |
+
tables = report.get("tables") if isinstance(report.get("tables"), dict) else {}
|
| 172 |
+
observed = {SYNC_KEY}
|
| 173 |
+
for table, row in tables.items():
|
| 174 |
+
state = str((row if isinstance(row, dict) else {}).get("state") or "done")
|
| 175 |
+
if state in AMAZON_GOOD or state in AMAZON_FAILED:
|
| 176 |
+
observed.add(str(table))
|
| 177 |
+
return observed
|
| 178 |
+
|
| 179 |
+
|
| 180 |
+
# ---- state ----------------------------------------------------------------------------------
|
| 181 |
+
|
| 182 |
+
def _alert_id(tenant, connector, key, code):
|
| 183 |
+
return json.dumps([tenant, connector, key, code])
|
| 184 |
+
|
| 185 |
+
|
| 186 |
+
def _scope(tenant, connector):
|
| 187 |
+
return json.dumps([tenant, connector])
|
| 188 |
+
|
| 189 |
+
|
| 190 |
+
def _due(alert, now):
|
| 191 |
+
"""The dedupe rule: what one alert needs this pass, or None."""
|
| 192 |
+
if not alert.get("sent_at"):
|
| 193 |
+
return "failed"
|
| 194 |
+
last = _parse(alert.get("reminded_at") or alert.get("sent_at"))
|
| 195 |
+
if last is None or now - last >= REMIND_AFTER:
|
| 196 |
+
return "reminder"
|
| 197 |
+
return None
|
| 198 |
+
|
| 199 |
+
|
| 200 |
+
def _reconcile(document, tenant, connector, current, observed, now, claim):
|
| 201 |
+
"""Fold this pass into the document; return the plan of what this pass must send."""
|
| 202 |
+
document = document if isinstance(document, dict) else {}
|
| 203 |
+
alerts = document.setdefault("alerts", {})
|
| 204 |
+
recovered = document.setdefault("recovered", {})
|
| 205 |
+
scopes = document.setdefault("scopes", {})
|
| 206 |
+
leases = document.setdefault("leases", {})
|
| 207 |
+
scope = _scope(tenant, connector)
|
| 208 |
+
state = scopes.setdefault(scope, {})
|
| 209 |
+
ok = state.setdefault("ok", {})
|
| 210 |
+
last_full = state.get("last_full") or {}
|
| 211 |
+
stamp = now.isoformat()
|
| 212 |
+
failing_keys = {f["key"] for f in current.values()}
|
| 213 |
+
|
| 214 |
+
for alert_id, failure in current.items():
|
| 215 |
+
alert = alerts.get(alert_id)
|
| 216 |
+
if alert is None and alert_id in recovered:
|
| 217 |
+
# Failing again before its RECOVERED went out: the owner never heard it cleared.
|
| 218 |
+
alert = {k: v for k, v in recovered.pop(alert_id).items() if k != "cleared_at"}
|
| 219 |
+
if alert is None:
|
| 220 |
+
last_good = failure["last_good"]
|
| 221 |
+
if not last_good:
|
| 222 |
+
candidates = [ok.get(failure["key"])]
|
| 223 |
+
if last_full.get("at") and failure["key"] not in (last_full.get("failing") or []):
|
| 224 |
+
candidates.append(last_full["at"])
|
| 225 |
+
candidates = [c for c in candidates if _parse(c)]
|
| 226 |
+
last_good = max(candidates, key=_parse) if candidates else None
|
| 227 |
+
alert = {"tenant": tenant, "connector": connector, "first": stamp, "last_good": last_good,
|
| 228 |
+
"sent_at": None, "reminded_at": None}
|
| 229 |
+
alert.update(key=failure["key"], label=failure["label"], code=failure["code"],
|
| 230 |
+
cause=failure["cause"])
|
| 231 |
+
alerts[alert_id] = alert
|
| 232 |
+
|
| 233 |
+
for alert_id in [a for a, v in alerts.items()
|
| 234 |
+
if v.get("tenant") == tenant and v.get("connector") == connector and a not in current]:
|
| 235 |
+
key = alerts[alert_id].get("key")
|
| 236 |
+
if observed is not None and key not in observed:
|
| 237 |
+
continue
|
| 238 |
+
alert = alerts.pop(alert_id)
|
| 239 |
+
ok[key] = stamp
|
| 240 |
+
if alert.get("sent_at"):
|
| 241 |
+
recovered[alert_id] = {**alert, "cleared_at": stamp}
|
| 242 |
+
|
| 243 |
+
if observed is None:
|
| 244 |
+
state["last_full"] = {"at": stamp, "failing": sorted(failing_keys)}
|
| 245 |
+
else:
|
| 246 |
+
for key in observed:
|
| 247 |
+
if key not in failing_keys:
|
| 248 |
+
ok[key] = stamp
|
| 249 |
+
if len(ok) > _OK_CAP:
|
| 250 |
+
state["ok"] = dict(sorted(ok.items(), key=lambda item: item[1])[-_OK_CAP:])
|
| 251 |
+
|
| 252 |
+
lease = leases.get(scope) or {}
|
| 253 |
+
held = lease.get("claim") not in (None, claim) and (_parse(lease.get("until")) or now) > now
|
| 254 |
+
if any(_pending(document, tenant, connector, now).values()) and not held:
|
| 255 |
+
leases[scope] = {"claim": claim, "until": (now + LEASE).isoformat()}
|
| 256 |
+
return document
|
| 257 |
+
|
| 258 |
+
|
| 259 |
+
def _pending(document, tenant, connector, now):
|
| 260 |
+
"""What this tenant and connector still owes the owner: unsent FAILED, due reminders, unsent RECOVERED."""
|
| 261 |
+
plan = {"failed": [], "reminder": [], "recovered": []}
|
| 262 |
+
for alert_id, alert in sorted((document.get("alerts") or {}).items()):
|
| 263 |
+
if alert.get("tenant") == tenant and alert.get("connector") == connector:
|
| 264 |
+
need = _due(alert, now)
|
| 265 |
+
if need:
|
| 266 |
+
plan[need].append(alert_id)
|
| 267 |
+
plan["recovered"] = sorted(a for a, v in (document.get("recovered") or {}).items()
|
| 268 |
+
if v.get("tenant") == tenant and v.get("connector") == connector)
|
| 269 |
+
return plan
|
| 270 |
+
|
| 271 |
+
|
| 272 |
+
def _plan_of(document, tenant, connector, now, claim):
|
| 273 |
+
lease = (document.get("leases") or {}).get(_scope(tenant, connector)) or {}
|
| 274 |
+
if lease.get("claim") != claim:
|
| 275 |
+
return None
|
| 276 |
+
return _pending(document, tenant, connector, now)
|
| 277 |
+
|
| 278 |
+
|
| 279 |
+
def _item(alert):
|
| 280 |
+
tail = (f"last good {when(alert['last_good'])}" if alert.get("last_good")
|
| 281 |
+
else f"failing since {when(alert.get('first'))}")
|
| 282 |
+
return f"{alert['label']} ({alert['cause']}, {tail})"
|
| 283 |
+
|
| 284 |
+
|
| 285 |
+
def environment():
|
| 286 |
+
"""LIVE, STAGING or LOCAL, from the Space id HF sets, so a staging alert never reads as live."""
|
| 287 |
+
space = (os.environ.get("SPACE_ID") or os.environ.get("APP_BASE_URL") or "").lower()
|
| 288 |
+
if "runloopable" in space:
|
| 289 |
+
return "LIVE"
|
| 290 |
+
return "STAGING" if "loopable" in space else "LOCAL"
|
| 291 |
+
|
| 292 |
+
|
| 293 |
+
def compose(tenant, connector, plan, document):
|
| 294 |
+
"""One line for the whole pass: FAILED, STILL FAILING and RECOVERED, each listed once."""
|
| 295 |
+
alerts, recovered = document.get("alerts") or {}, document.get("recovered") or {}
|
| 296 |
+
where = f"{environment()} {tenant} {connector}"
|
| 297 |
+
parts = []
|
| 298 |
+
for kind, title in (("failed", "FAILED"), ("reminder", "STILL FAILING")):
|
| 299 |
+
items = [_item(alerts[a]) for a in sorted(plan[kind]) if a in alerts]
|
| 300 |
+
if items:
|
| 301 |
+
parts.append(f"Loopable sync {title}: {where}: " + ", ".join(items) + ".")
|
| 302 |
+
names = [recovered[a]["label"] for a in plan["recovered"] if a in recovered]
|
| 303 |
+
if names:
|
| 304 |
+
parts.append(f"Loopable sync RECOVERED: {where}: " + ", ".join(names) + ".")
|
| 305 |
+
text = " ".join(parts)
|
| 306 |
+
return text if len(text) <= MESSAGE_CAP else text[:MESSAGE_CAP - 3].rstrip() + "..."
|
| 307 |
+
|
| 308 |
+
|
| 309 |
+
def _mark(document, tenant, connector, plan, now, claim, delivered):
|
| 310 |
+
document = document if isinstance(document, dict) else {}
|
| 311 |
+
alerts = document.setdefault("alerts", {})
|
| 312 |
+
recovered = document.setdefault("recovered", {})
|
| 313 |
+
leases = document.setdefault("leases", {})
|
| 314 |
+
scope = _scope(tenant, connector)
|
| 315 |
+
if (leases.get(scope) or {}).get("claim") == claim:
|
| 316 |
+
leases.pop(scope, None)
|
| 317 |
+
if delivered:
|
| 318 |
+
stamp = now.isoformat()
|
| 319 |
+
for alert_id in plan["failed"]:
|
| 320 |
+
if alert_id in alerts:
|
| 321 |
+
alerts[alert_id].update(sent_at=stamp, reminded_at=stamp)
|
| 322 |
+
for alert_id in plan["reminder"]:
|
| 323 |
+
if alert_id in alerts:
|
| 324 |
+
alerts[alert_id]["reminded_at"] = stamp
|
| 325 |
+
for alert_id in plan["recovered"]:
|
| 326 |
+
recovered.pop(alert_id, None)
|
| 327 |
+
return document
|
| 328 |
+
|
| 329 |
+
|
| 330 |
+
# ---- entry point ----------------------------------------------------------------------------
|
| 331 |
+
|
| 332 |
+
def notify(tenant, connector, failures, *, observed=None, now=None, sender=None, store=None):
|
| 333 |
+
"""Record this pass's failures and send at most one Telegram line; never raises.
|
| 334 |
+
|
| 335 |
+
`failures` is the full current list for the tenant and connector; [] means healthy.
|
| 336 |
+
`observed` limits clearing to the keys this pass checked; None means the list is complete.
|
| 337 |
+
"""
|
| 338 |
+
result = {"ok": False, "sent": False, "failed": [], "reminder": [], "recovered": [], "cause": None}
|
| 339 |
+
try:
|
| 340 |
+
now = (now or utc_now()).astimezone(timezone.utc)
|
| 341 |
+
tenant, connector = str(tenant), str(connector)
|
| 342 |
+
store = store if store is not None else store_backend()
|
| 343 |
+
current = {}
|
| 344 |
+
for failure in failures or []:
|
| 345 |
+
clean = _clean(failure)
|
| 346 |
+
if clean:
|
| 347 |
+
current[_alert_id(tenant, connector, clean["key"], clean["code"])] = clean
|
| 348 |
+
observed = None if observed is None else {str(key) for key in observed}
|
| 349 |
+
claim = uuid4().hex
|
| 350 |
+
# Save first, so a delivery marker from a past pass is never re-sent.
|
| 351 |
+
document = store.update(DOCUMENT, lambda doc: _reconcile(
|
| 352 |
+
doc, tenant, connector, current, observed, now, claim))
|
| 353 |
+
plan = _plan_of(document or {}, tenant, connector, now, claim)
|
| 354 |
+
if plan is None or not any(plan.values()):
|
| 355 |
+
result.update(ok=True, cause=None if plan is not None else "another pass is sending")
|
| 356 |
+
return result
|
| 357 |
+
text = compose(tenant, connector, plan, document)
|
| 358 |
+
try:
|
| 359 |
+
response = (sender or default_sender)(text)
|
| 360 |
+
delivered, detail = (response if isinstance(response, tuple) and len(response) == 2
|
| 361 |
+
else (response is not False, None))
|
| 362 |
+
except Exception: # noqa: BLE001
|
| 363 |
+
delivered, detail = False, "Telegram alert could not be sent."
|
| 364 |
+
delivered = bool(delivered)
|
| 365 |
+
store.update(DOCUMENT, lambda doc: _mark(doc, tenant, connector, plan, now, claim, delivered))
|
| 366 |
+
result.update({kind: list(ids) for kind, ids in plan.items()})
|
| 367 |
+
result.update(ok=delivered, sent=delivered,
|
| 368 |
+
cause=None if delivered else scrub(detail or "Telegram alert was not delivered."))
|
| 369 |
+
return result
|
| 370 |
+
except Exception as exc: # noqa: BLE001
|
| 371 |
+
result["cause"] = f"sync alert skipped ({type(exc).__name__})"
|
| 372 |
+
return result
|
web/public/AIOS_DEPLOY_RECEIPT
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
|
|
|
|
| 1 |
+
f8fbaa9615aa924d504ee1421880fa42358ace44cae3257345d88081cc31bacb
|
web/src/connectors/OdooConfig.tsx
CHANGED
|
@@ -92,7 +92,7 @@ export interface OdooFreshness {
|
|
| 92 |
failure?: { code?: string; message?: string; at?: string | null } | null;
|
| 93 |
/** D-2679: databases that failed while the rest of the sync succeeded. */
|
| 94 |
failedDatabases?: { key: string; label?: string | null; code?: string | null; message?: string | null;
|
| 95 |
-
stage?: string | null; at?: string | null }[];
|
| 96 |
syncAt?: string | null;
|
| 97 |
syncTz?: string | null;
|
| 98 |
nextSyncAt?: string | null;
|
|
@@ -149,7 +149,9 @@ export function freshnessWarnings(f: OdooFreshness | null | undefined): string[]
|
|
| 149 |
for (const db of Array.isArray(f.failedDatabases) ? f.failedDatabases : []) {
|
| 150 |
const label = (db?.label || db?.key || "").trim();
|
| 151 |
const message = (db?.message ?? "").trim();
|
| 152 |
-
|
|
|
|
|
|
|
| 153 |
}
|
| 154 |
return lines;
|
| 155 |
}
|
|
|
|
| 92 |
failure?: { code?: string; message?: string; at?: string | null } | null;
|
| 93 |
/** D-2679: databases that failed while the rest of the sync succeeded. */
|
| 94 |
failedDatabases?: { key: string; label?: string | null; code?: string | null; message?: string | null;
|
| 95 |
+
stage?: string | null; at?: string | null; cause?: string | null }[];
|
| 96 |
syncAt?: string | null;
|
| 97 |
syncTz?: string | null;
|
| 98 |
nextSyncAt?: string | null;
|
|
|
|
| 149 |
for (const db of Array.isArray(f.failedDatabases) ? f.failedDatabases : []) {
|
| 150 |
const label = (db?.label || db?.key || "").trim();
|
| 151 |
const message = (db?.message ?? "").trim();
|
| 152 |
+
// The cause stays on the same line: failedDatabasesStatus reads every line as a database.
|
| 153 |
+
const cause = (db?.cause ?? "").trim();
|
| 154 |
+
if (label) lines.push(`${label}${FAILED_DATABASE_LINE}${message ? ` ${message}` : ""}${cause ? ` Cause: ${cause}` : ""}`);
|
| 155 |
}
|
| 156 |
return lines;
|
| 157 |
}
|