fsanyoto commited on
Commit
89d8428
·
verified ·
1 Parent(s): bc87642

Deploy AIOS web (React glide grid + FastAPI slice)

Browse files
RELEASES.json CHANGED
@@ -1,5 +1,5 @@
1
  {
2
- "current": "95f9e8c (95f9e8cec7d1)",
3
  "releases": [
4
  {
5
  "version": "v86",
 
1
  {
2
+ "current": "eda4a2ee5cf5 (eda4a2ee5cf5)",
3
  "releases": [
4
  {
5
  "version": "v86",
VERSION CHANGED
@@ -1 +1 @@
1
- 95f9e8c (95f9e8cec7d1)
 
1
+ eda4a2ee5cf5 (eda4a2ee5cf5)
api/main.py CHANGED
@@ -1111,6 +1111,13 @@ def _pull_amazon(why, vendor=True):
1111
  print(f"[aios-api] amazon sync done ({why}): " + ", ".join(
1112
  f"{table}={row.get('state', 'done')}" + (f", {row['rows']} rows" if 'rows' in row else "")
1113
  for table, row in rep["tables"].items()))
 
 
 
 
 
 
 
1114
  return not bool(rep.get("problems"))
1115
  except Exception as exc: # noqa: BLE001
1116
  print(f"[aios-api] amazon sync FAILED ({why}): {type(exc).__name__}: {exc}")
@@ -2070,13 +2077,22 @@ def _rebuild_odoo_relational(why, previous_meta_fingerprint=None, meta_state_out
2070
  # one this mirror serves. Best effort: a stamp must never fail the rebuild it records.
2071
  if cid == str(default_cid):
2072
  # D-2679: a good sync still names the databases that failed by themselves.
 
 
 
 
 
 
 
 
2073
  failed_databases = None if safe_errors else [
2074
  {"key": key, "code": db_failure.get("code"),
2075
  "stage": db_failure.get("stage"), "at": refreshed_at,
2076
  "message": routes_connectors._DATABASE_SAFE_ERRORS.get(
2077
  str(db_failure.get("code") or ""))
2078
  or routes_connectors._odoo_failure(
2079
- None, str(db_failure.get("stage") or "materialization"))["message"]}
 
2080
  for key, db_failure in blocking.items() if isinstance(db_failure, dict)]
2081
  if failed_databases is not None and table_keys is not None:
2082
  # D-2679: a changed-tables pass keeps the failures of databases it did not run.
@@ -2092,6 +2108,13 @@ def _rebuild_odoo_relational(why, previous_meta_fingerprint=None, meta_state_out
2092
  failed_databases=failed_databases)
2093
  except Exception: # noqa: BLE001
2094
  pass
 
 
 
 
 
 
 
2095
  meta_state = _rebuild_meta_relational(
2096
  why, _rt, previous_fingerprint=previous_meta_fingerprint, force_relations=True)
2097
  if isinstance(meta_state_out, dict):
 
1111
  print(f"[aios-api] amazon sync done ({why}): " + ", ".join(
1112
  f"{table}={row.get('state', 'done')}" + (f", {row['rows']} rows" if 'rows' in row else "")
1113
  for table, row in rep["tables"].items()))
1114
+ # Owner 2026-10-29: an Amazon table that fails or recovers is Telegrammed at once.
1115
+ try:
1116
+ import sync_alerts as _sa
1117
+ _sa.notify(tenant_key, "amazon", _sa.amazon_failures(rep), observed=_sa.amazon_observed(rep),
1118
+ store=rt)
1119
+ except Exception: # noqa: BLE001
1120
+ pass
1121
  return not bool(rep.get("problems"))
1122
  except Exception as exc: # noqa: BLE001
1123
  print(f"[aios-api] amazon sync FAILED ({why}): {type(exc).__name__}: {exc}")
 
2077
  # one this mirror serves. Best effort: a stamp must never fail the rebuild it records.
2078
  if cid == str(default_cid):
2079
  # D-2679: a good sync still names the databases that failed by themselves.
2080
+ # The health cleaner drops `cause`, so it is read from the raw report entry.
2081
+ def _raw_cause(key):
2082
+ receipt = all_receipts.get(key) if isinstance(all_receipts.get(key), dict) else {}
2083
+ for raw in ((report.get("databaseErrors") or {}).get(key),
2084
+ receipt.get("error"), receipt.get("sourceError")):
2085
+ if isinstance(raw, dict) and raw.get("cause"):
2086
+ return str(raw["cause"])
2087
+ return None
2088
  failed_databases = None if safe_errors else [
2089
  {"key": key, "code": db_failure.get("code"),
2090
  "stage": db_failure.get("stage"), "at": refreshed_at,
2091
  "message": routes_connectors._DATABASE_SAFE_ERRORS.get(
2092
  str(db_failure.get("code") or ""))
2093
  or routes_connectors._odoo_failure(
2094
+ None, str(db_failure.get("stage") or "materialization"))["message"],
2095
+ **({"cause": _raw_cause(key)} if _raw_cause(key) else {})}
2096
  for key, db_failure in blocking.items() if isinstance(db_failure, dict)]
2097
  if failed_databases is not None and table_keys is not None:
2098
  # D-2679: a changed-tables pass keeps the failures of databases it did not run.
 
2108
  failed_databases=failed_databases)
2109
  except Exception: # noqa: BLE001
2110
  pass
2111
+ # Owner 2026-10-29: a database that fails or recovers is Telegrammed at once.
2112
+ try:
2113
+ import sync_alerts as _sa
2114
+ _sa.notify(tenant_key, "odoo", _sa.odoo_failures(failed_databases, safe_errors),
2115
+ observed=_sa.odoo_observed(failed_databases), store=_rt)
2116
+ except Exception: # noqa: BLE001
2117
+ pass
2118
  meta_state = _rebuild_meta_relational(
2119
  why, _rt, previous_fingerprint=previous_meta_fingerprint, force_relations=True)
2120
  if isinstance(meta_state_out, dict):
api/odoo_relational.py CHANGED
@@ -3951,6 +3951,14 @@ def capture_source_counts(rt, connection_id=None, buckets=None):
3951
  counts[bucket] = _source_domain_count(client, bucket)
3952
  except Exception: # noqa: BLE001
3953
  continue
 
 
 
 
 
 
 
 
3954
  return counts
3955
 
3956
 
@@ -5700,7 +5708,8 @@ def record_sync_outcome(rt, ok, at=None, failed_databases=None):
5700
  "message": _plain(item.get("message")
5701
  or "The database could not be materialized.", 300),
5702
  "stage": _plain(item.get("stage") or "materialization", 40),
5703
- "at": str(item.get("at") or stamp)})
 
5704
 
5705
  def _up(cur):
5706
  cur = dict(cur) if isinstance(cur, dict) else {}
@@ -5925,7 +5934,8 @@ def sync_freshness(rt, now=None):
5925
  failed_databases.append({
5926
  name: (" ".join(str(item.get(name) or "").replace("—", ",")
5927
  .replace("–", ",").split())[:300] or None)
5928
- for name in ("key", "label", "code", "message", "stage", "at")})
 
5929
  return {"stale": bool(stale), "lastGoodSyncAt": last_good,
5930
  "failedDatabases": failed_databases,
5931
  "schedule": cfg["syncEvery"], "staleAfterSeconds": allowed,
@@ -6182,8 +6192,25 @@ def read_shipments_live(client):
6182
  return rows
6183
 
6184
 
6185
- def reconcile_shipments(rows, client, year):
6186
- """Independent source count for completed outgoing shipments in one UTC calendar year."""
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6187
  year = int(year)
6188
  start, end = f"{year:04d}-01-01", f"{year + 1:04d}-01-01"
6189
  if hasattr(rows, "execute"):
@@ -6197,11 +6224,14 @@ def reconcile_shipments(rows, client, year):
6197
  values = rows.values() if isinstance(rows, dict) else rows
6198
  count = sum(1 for row in values if row.get("state") == "done"
6199
  and start <= (row.get("date_done") or "") < end)
6200
- source_count = client.search_count("stock.picking", [
6201
- ("picking_type_code", "=", "outgoing"), ("state", "=", "done"),
6202
- ("date_done", ">=", start + " 00:00:00"), ("date_done", "<", end + " 00:00:00")])
6203
- return {"year": year, "rowCount": count, "sourceCount": source_count,
6204
- "ok": type(source_count) is int and source_count == count}
 
 
 
6205
 
6206
 
6207
  def shipment_source_spec():
@@ -6310,8 +6340,11 @@ def apply_shipments(rt, planned, username="automation", today=None):
6310
 
6311
 
6312
  def refresh_shipments(rt, *, connection_id=None, year=None, today=None,
6313
- username="automation", client=None, cur=None):
6314
- """Reconcile the tenant's shipment mirror against an independent Odoo count."""
 
 
 
6315
  if client is None:
6316
  source = rt.odoo_source() if connection_id is None else rt.odoo_source(connection_id)
6317
  if source is None:
@@ -6319,11 +6352,12 @@ def refresh_shipments(rt, *, connection_id=None, year=None, today=None,
6319
  from core.odoo import get_odoo
6320
  with source._bound():
6321
  return refresh_shipments(rt, connection_id=connection_id, year=year, today=today,
6322
- username=username, client=get_odoo(), cur=cur)
 
6323
  stamp = today or _iso_today()
6324
  year = int(year) if year is not None else int(stamp[:4])
6325
  planned = plan_shipments(rt, connection_id, client, stamp, cur)
6326
- reconciled = reconcile_shipments(planned["rows"], client, year)
6327
  if not reconciled["ok"]:
6328
  raise Refused(f"stock.picking: {year} done outgoing shipments do not reconcile "
6329
  f"({reconciled['rowCount']} mirror, {reconciled['sourceCount']} source)")
@@ -7032,6 +7066,48 @@ def health_view(report):
7032
  return view
7033
 
7034
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
7035
  def refresh_curated(rt, cur, buckets, *, connection_id=None, username="automation",
7036
  today=None, retire=True, on_apply=None, source_client=None,
7037
  source_counts=None):
@@ -7062,7 +7138,9 @@ def refresh_curated(rt, cur, buckets, *, connection_id=None, username="automatio
7062
  source_failure = {
7063
  "code": "source_reconciliation_failed", "type": type(exc).__name__,
7064
  "message": "This Odoo database source population could not be counted.",
 
7065
  }
 
7066
  identity = {
7067
  "databaseKey": key,
7068
  "schemaKey": key,
@@ -7130,10 +7208,12 @@ def refresh_curated(rt, cur, buckets, *, connection_id=None, username="automatio
7130
  if source_failure:
7131
  errors[key] = source_failure
7132
  except Exception as exc:
7133
- # Exception prose may contain connection or record values. Keep only the
7134
- # structural class and a fixed product message in returned/persisted evidence.
7135
  failure = {"code": "materialization_failed", "type": type(exc).__name__,
7136
- "message": "This Odoo database could not be materialized."}
 
 
7137
  errors[key] = failure
7138
  receipts[key] = {
7139
  **identity, "state": "failed", "sourceCount": source_count,
@@ -7296,16 +7376,22 @@ def refresh(rt, tenant, username="automation", cur=None, today=None, connection_
7296
  key = table_key_for(SHIPMENTS_KEY, ns)
7297
  year = int((today or _iso_today())[:4])
7298
  receipt = database_receipts.setdefault(key, {"databaseKey": key})
 
 
 
7299
  try:
7300
  shipment_report = refresh_shipments(
7301
  rt, connection_id=connection_id, year=year, today=today,
7302
- username=username, cur=cur)
 
7303
  receipt["shipmentReconciliation"] = shipment_report["reconcile_mirror"]
7304
  except Exception as exc: # noqa: BLE001
7305
  # A source refusal belongs to this database and must not abort the pass.
7306
  failure = {"code": "source_reconciliation_failed", "type": type(exc).__name__,
7307
  "message": f"Completed outgoing shipments could not be reconciled "
7308
- f"with Odoo for {year}."}
 
 
7309
  database_errors.setdefault(key, failure)
7310
  receipt["shipmentReconciliation"] = {"year": year, "ok": False}
7311
  receipt["sourceError"] = failure
 
3951
  counts[bucket] = _source_domain_count(client, bucket)
3952
  except Exception: # noqa: BLE001
3953
  continue
3954
+ if "shipments" in chosen:
3955
+ # The year's done count reconciles ut_odoo_shipments against the pull, not a later Odoo.
3956
+ year = int(_iso_today()[:4])
3957
+ try:
3958
+ counts[shipments_done_count_key(year)] = (
3959
+ int(_shipments_done_source_count(client, year)), "search_count")
3960
+ except Exception: # noqa: BLE001
3961
+ pass
3962
  return counts
3963
 
3964
 
 
5708
  "message": _plain(item.get("message")
5709
  or "The database could not be materialized.", 300),
5710
  "stage": _plain(item.get("stage") or "materialization", 40),
5711
+ "at": str(item.get("at") or stamp),
5712
+ **({"cause": _plain(item["cause"], 300)} if item.get("cause") else {})})
5713
 
5714
  def _up(cur):
5715
  cur = dict(cur) if isinstance(cur, dict) else {}
 
5934
  failed_databases.append({
5935
  name: (" ".join(str(item.get(name) or "").replace("—", ",")
5936
  .replace("–", ",").split())[:300] or None)
5937
+ for name in ("key", "label", "code", "message", "stage", "at")
5938
+ + (("cause",) if item.get("cause") else ())})
5939
  return {"stale": bool(stale), "lastGoodSyncAt": last_good,
5940
  "failedDatabases": failed_databases,
5941
  "schedule": cfg["syncEvery"], "staleAfterSeconds": allowed,
 
6192
  return rows
6193
 
6194
 
6195
+ def shipments_done_count_key(year):
6196
+ """The `capture_source_counts` key for one year's completed outgoing shipment count."""
6197
+ return f"shipments_done_{int(year):04d}"
6198
+
6199
+
6200
+ def _shipments_done_source_count(client, year):
6201
+ """Odoo's count of completed outgoing shipments in one naive UTC calendar year."""
6202
+ year = int(year)
6203
+ start, end = f"{year:04d}-01-01", f"{year + 1:04d}-01-01"
6204
+ return client.search_count("stock.picking", [
6205
+ ("picking_type_code", "=", "outgoing"), ("state", "=", "done"),
6206
+ ("date_done", ">=", start + " 00:00:00"), ("date_done", "<", end + " 00:00:00")])
6207
+
6208
+
6209
+ def reconcile_shipments(rows, client, year, source_count=None):
6210
+ """Independent source count for completed outgoing shipments in one UTC calendar year.
6211
+
6212
+ `source_count` is the count taken at mirror-pull time. Without it Odoo is counted now.
6213
+ """
6214
  year = int(year)
6215
  start, end = f"{year:04d}-01-01", f"{year + 1:04d}-01-01"
6216
  if hasattr(rows, "execute"):
 
6224
  values = rows.values() if isinstance(rows, dict) else rows
6225
  count = sum(1 for row in values if row.get("state") == "done"
6226
  and start <= (row.get("date_done") or "") < end)
6227
+ captured = source_count is not None
6228
+ if not captured:
6229
+ source_count = _shipments_done_source_count(client, year)
6230
+ result = {"year": year, "rowCount": count, "sourceCount": source_count,
6231
+ "ok": type(source_count) is int and source_count == count}
6232
+ if captured:
6233
+ result["sourceCountMode"] = "pull_time"
6234
+ return result
6235
 
6236
 
6237
  def shipment_source_spec():
 
6340
 
6341
 
6342
  def refresh_shipments(rt, *, connection_id=None, year=None, today=None,
6343
+ username="automation", client=None, cur=None, source_count=None):
6344
+ """Reconcile the tenant's shipment mirror against an independent Odoo count.
6345
+
6346
+ `source_count` is the year's done count taken at mirror-pull time, if one was captured.
6347
+ """
6348
  if client is None:
6349
  source = rt.odoo_source() if connection_id is None else rt.odoo_source(connection_id)
6350
  if source is None:
 
6352
  from core.odoo import get_odoo
6353
  with source._bound():
6354
  return refresh_shipments(rt, connection_id=connection_id, year=year, today=today,
6355
+ username=username, client=get_odoo(), cur=cur,
6356
+ source_count=source_count)
6357
  stamp = today or _iso_today()
6358
  year = int(year) if year is not None else int(stamp[:4])
6359
  planned = plan_shipments(rt, connection_id, client, stamp, cur)
6360
+ reconciled = reconcile_shipments(planned["rows"], client, year, source_count=source_count)
6361
  if not reconciled["ok"]:
6362
  raise Refused(f"stock.picking: {year} done outgoing shipments do not reconcile "
6363
  f"({reconciled['rowCount']} mirror, {reconciled['sourceCount']} source)")
 
7066
  return view
7067
 
7068
 
7069
+ import re as _re_scrub # noqa: E402 (kept beside the scrubber it serves)
7070
+
7071
+ _SCRUB_RULES = (
7072
+ (_re_scrub.compile(r"\b[a-z][a-z0-9+.\-]*://\S+", _re_scrub.I), "<url>"),
7073
+ (_re_scrub.compile(r"[\w.+\-]+@[\w\-]+(?:\.[\w\-]+)+"), "<email>"),
7074
+ (_re_scrub.compile(r"\bbearer\s+\S+", _re_scrub.I), "Bearer <redacted>"),
7075
+ (_re_scrub.compile(r"\b(password|passwd|pwd|secret|token|api[_\-]?key|apikey|authorization"
7076
+ r"|session[_\-]?id|cookie|access[_\-]?key|client[_\-]?secret)\b"
7077
+ r"([\"']?\s*[:=]\s*)(\"[^\"]*\"|'[^']*'|\S+)", _re_scrub.I),
7078
+ r"\1\2<redacted>"),
7079
+ (_re_scrub.compile(r"\b\d{1,3}(?:\.\d{1,3}){3}(?::\d+)?\b"), "<host>"),
7080
+ (_re_scrub.compile(r"\b(?=[A-Za-z0-9_\-]*\d)(?=[A-Za-z0-9_\-]*[A-Za-z])[A-Za-z0-9_\-]{20,}\b"),
7081
+ "<token>"),
7082
+ (_re_scrub.compile(r"=\([^)]*\)"), "=(<value>)"),
7083
+ (_re_scrub.compile(r"\d{9,}"), "<number>"),
7084
+ )
7085
+ _SCRUB_QUOTED = _re_scrub.compile(r"'([^'\n]*)'|\"([^\"\n]*)\"")
7086
+ _SCRUB_NAME = _re_scrub.compile(r"[A-Za-z_][A-Za-z0-9_.]*")
7087
+
7088
+
7089
+ def _scrub_cause(exc):
7090
+ """`Type: message` for a log and a card, with credentials and record values masked, 300 chars.
7091
+
7092
+ Names (models, tables, columns) and counts stay. Quoted free text is a record value and goes.
7093
+ """
7094
+ name = type(exc).__name__ if isinstance(exc, BaseException) else "Error"
7095
+ text = str(exc if exc is not None else "")
7096
+ for pattern, mask in _SCRUB_RULES:
7097
+ text = pattern.sub(mask, text)
7098
+ text = _SCRUB_QUOTED.sub(
7099
+ lambda m: m.group(0) if _SCRUB_NAME.fullmatch(m.group(1) or m.group(2) or "")
7100
+ else "'<value>'", text)
7101
+ text = " ".join(text.replace(chr(0x2014), ",").replace(chr(0x2013), ",").split())
7102
+ return (f"{name}: {text}" if text else name)[:300]
7103
+
7104
+
7105
+ def _log_database_failure(key, failure):
7106
+ """One log line naming a failed database, its code and its scrubbed cause."""
7107
+ print(f"[aios-api] odoo database failed: {key} {failure.get('code')} {failure.get('cause')}",
7108
+ flush=True)
7109
+
7110
+
7111
  def refresh_curated(rt, cur, buckets, *, connection_id=None, username="automation",
7112
  today=None, retire=True, on_apply=None, source_client=None,
7113
  source_counts=None):
 
7138
  source_failure = {
7139
  "code": "source_reconciliation_failed", "type": type(exc).__name__,
7140
  "message": "This Odoo database source population could not be counted.",
7141
+ "cause": _scrub_cause(exc),
7142
  }
7143
+ _log_database_failure(key, source_failure)
7144
  identity = {
7145
  "databaseKey": key,
7146
  "schemaKey": key,
 
7208
  if source_failure:
7209
  errors[key] = source_failure
7210
  except Exception as exc:
7211
+ # Exception prose may contain connection or record values: only the scrubbed cause
7212
+ # carries it, beside the structural class and a fixed product message.
7213
  failure = {"code": "materialization_failed", "type": type(exc).__name__,
7214
+ "message": "This Odoo database could not be materialized.",
7215
+ "cause": _scrub_cause(exc)}
7216
+ _log_database_failure(key, failure)
7217
  errors[key] = failure
7218
  receipts[key] = {
7219
  **identity, "state": "failed", "sourceCount": source_count,
 
7376
  key = table_key_for(SHIPMENTS_KEY, ns)
7377
  year = int((today or _iso_today())[:4])
7378
  receipt = database_receipts.setdefault(key, {"databaseKey": key})
7379
+ # Pull-time counts describe the default connection's mirror only.
7380
+ captured = ((source_counts or {}).get(shipments_done_count_key(year))
7381
+ if ns is None else None)
7382
  try:
7383
  shipment_report = refresh_shipments(
7384
  rt, connection_id=connection_id, year=year, today=today,
7385
+ username=username, cur=cur,
7386
+ **({"source_count": captured[0]} if captured else {}))
7387
  receipt["shipmentReconciliation"] = shipment_report["reconcile_mirror"]
7388
  except Exception as exc: # noqa: BLE001
7389
  # A source refusal belongs to this database and must not abort the pass.
7390
  failure = {"code": "source_reconciliation_failed", "type": type(exc).__name__,
7391
  "message": f"Completed outgoing shipments could not be reconciled "
7392
+ f"with Odoo for {year}.",
7393
+ "cause": _scrub_cause(exc)}
7394
+ _log_database_failure(key, failure)
7395
  database_errors.setdefault(key, failure)
7396
  receipt["shipmentReconciliation"] = {"year": year, "ok": False}
7397
  receipt["sourceError"] = failure
api/routes_connectors.py CHANGED
@@ -354,6 +354,7 @@ _DATABASE_SAFE_ERRORS = {
354
  "mirror_pull_failed": "The Odoo mirror pull failed.",
355
  "materialization_failed": "The database could not be materialized.",
356
  "reconciliation_failed": "The database reconciliation failed.",
 
357
  "freshness_unconfirmed": "Freshness could not be confirmed for this database.",
358
  "sync_status_unavailable": "Sync status is unavailable for this database.",
359
  "sync_stalled": "This database sync is stalled.",
 
354
  "mirror_pull_failed": "The Odoo mirror pull failed.",
355
  "materialization_failed": "The database could not be materialized.",
356
  "reconciliation_failed": "The database reconciliation failed.",
357
+ "source_reconciliation_failed": "Its row count could not be matched to Odoo.",
358
  "freshness_unconfirmed": "Freshness could not be confirmed for this database.",
359
  "sync_status_unavailable": "Sync status is unavailable for this database.",
360
  "sync_stalled": "This database sync is stalled.",
api/sync_alerts.py ADDED
@@ -0,0 +1,372 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Tell the owner on Telegram, once, when a connector database fails, recovers or stays failed for a day."""
2
+ from __future__ import annotations
3
+
4
+ from datetime import datetime, timedelta, timezone
5
+ import json
6
+ import os
7
+ from pathlib import Path
8
+ import re
9
+ import sys
10
+ from uuid import uuid4
11
+
12
+ DOCUMENT = "ops_sync_alerts"
13
+ REMIND_AFTER = timedelta(hours=24)
14
+ LEASE = timedelta(minutes=5)
15
+ CAUSE_CAP = 300
16
+ MESSAGE_CAP = 1500
17
+ _OK_CAP = 200
18
+ # Amazon table states that are a failure, and the one state that proves a table healthy.
19
+ # `throttled` is left out: Amazon asked us to wait and the next pass retries it, so it is not a failure.
20
+ AMAZON_FAILED = frozenset({"refused", "failed", "error"})
21
+ AMAZON_GOOD = frozenset({"done"})
22
+ SYNC_KEY = "sync"
23
+
24
+
25
+ def utc_now():
26
+ return datetime.now(timezone.utc)
27
+
28
+
29
+ def store_backend():
30
+ sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "platform"))
31
+ import core.store as store
32
+ return store
33
+
34
+
35
+ def default_sender(text):
36
+ sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "platform"))
37
+ from core.neon_usage import send_telegram
38
+ return send_telegram(text)
39
+
40
+
41
+ # ---- text -----------------------------------------------------------------------------------
42
+
43
+ _SCRUB = [
44
+ (re.compile(r"(?i)\b([a-z][a-z0-9+.-]*://)[^/\s:@]+:[^/\s@]+@"), r"\1[credentials]@"),
45
+ (re.compile(r"[\w.+-]+@[\w-]+(?:\.[\w-]+)+"), "[email]"),
46
+ (re.compile(r"(?i)\bbearer\s+[A-Za-z0-9._~+/=-]+"), "Bearer [token]"),
47
+ (re.compile(r"\bAtz[ar]\|[^\s,;\"']+"), "[token]"),
48
+ (re.compile(r"(?i)\b(bot)\d+:[A-Za-z0-9_-]+"), r"\1[token]"),
49
+ (re.compile(r"(?i)((?:^|[?&;\s])(?:access_token|refresh_token|token|key|api_key|apikey|secret|"
50
+ r"client_secret|password|passwd|pwd|signature|sig)=)[^&\s\"']+"), r"\1[masked]"),
51
+ (re.compile(r"(?i)\b((?:access_token|refresh_token|client_secret|password|secret|api_key)"
52
+ r"[\"']?\s*[:=]\s*[\"']?)[^\s,;\"'}]+"), r"\1[masked]"),
53
+ # A long run holding a digit and a letter is a key; plain snake_case table names are not.
54
+ (re.compile(r"\b(?=[\w\-+/=]*\d)(?=[\w\-+/=]*[A-Za-z])[\w\-+/=]{32,}"), "[token]"),
55
+ ]
56
+
57
+
58
+ def scrub(text, cap=CAUSE_CAP):
59
+ """Mask credentials, tokens and emails, drop dashes the owner must not read, and cap the length."""
60
+ text = str(text if text is not None else "").replace("\u2014", ", ").replace("\u2013", " to ")
61
+ text = re.sub(r"\s+([,.;:])", r"\1", " ".join(text.split()))
62
+ for pattern, replacement in _SCRUB:
63
+ text = pattern.sub(replacement, text)
64
+ return text if len(text) <= cap else text[:cap - 3].rstrip() + "..."
65
+
66
+
67
+ def _parse(iso):
68
+ try:
69
+ value = datetime.fromisoformat(str(iso).replace("Z", "+00:00"))
70
+ except (TypeError, ValueError):
71
+ return None
72
+ return value.replace(tzinfo=timezone.utc) if value.tzinfo is None else value.astimezone(timezone.utc)
73
+
74
+
75
+ def _nth_sunday(year, month, n):
76
+ first = datetime(year, month, 1, tzinfo=timezone.utc)
77
+ return first + timedelta(days=(6 - first.weekday()) % 7 + 7 * (n - 1))
78
+
79
+
80
+ def _new_york_offset(moment):
81
+ # EDT from the second Sunday of March 2 AM EST to the first Sunday of November 2 AM EDT.
82
+ start = _nth_sunday(moment.year, 3, 2) + timedelta(hours=7)
83
+ end = _nth_sunday(moment.year, 11, 1) + timedelta(hours=6)
84
+ return timedelta(hours=-4) if start <= moment < end else timedelta(hours=-5)
85
+
86
+
87
+ def _clock(moment):
88
+ hour = moment.hour % 12 or 12
89
+ return f"{moment.day} {moment.strftime('%b')} {hour}:{moment.minute:02d} {'AM' if moment.hour < 12 else 'PM'}"
90
+
91
+
92
+ def when(iso):
93
+ """One instant as Jakarta and New York time, side by side; never a bare UTC time."""
94
+ moment = _parse(iso)
95
+ if moment is None:
96
+ return "an unknown time"
97
+ jakarta = moment + timedelta(hours=7)
98
+ new_york = moment + _new_york_offset(moment)
99
+ return f"{_clock(jakarta)} Jakarta / {_clock(new_york)} New York"
100
+
101
+
102
+ # ---- failure lists --------------------------------------------------------------------------
103
+
104
+ def _clean(failure):
105
+ if not isinstance(failure, dict) or not str(failure.get("key") or "").strip():
106
+ return None
107
+ key = scrub(failure["key"], 120)
108
+ return {"key": key, "label": scrub(failure.get("label") or key, 120),
109
+ "code": scrub(failure.get("code") or "error", 60) or "error",
110
+ "cause": scrub(failure.get("cause") or "no cause given"),
111
+ "last_good": failure.get("last_good") if _parse(failure.get("last_good")) else None}
112
+
113
+
114
+ def odoo_failures(failed_databases=None, safe_errors=None):
115
+ """Odoo's per-database failures, or the whole connection's errors when the pass failed outright."""
116
+ out = []
117
+ for item in failed_databases or []:
118
+ if isinstance(item, dict) and item.get("key"):
119
+ out.append({"key": str(item["key"]), "label": str(item.get("label") or item["key"]),
120
+ "code": str(item.get("code") or item.get("stage") or "error"),
121
+ "cause": str(item.get("cause") or item.get("message")
122
+ or "the database did not sync")})
123
+ for text in safe_errors or []:
124
+ code, _, cause = str(text).partition(":")
125
+ out.append({"key": "connection", "label": "Odoo connection", "code": code.strip() or "error",
126
+ "cause": cause.strip() or code.strip()})
127
+ return out
128
+
129
+
130
+ def odoo_observed(failed_databases=None):
131
+ """A failed connection checked no database, so per-database alerts carry forward; None means a full list."""
132
+ return None if failed_databases is not None else {"connection"}
133
+
134
+
135
+ def amazon_failures(report):
136
+ """Amazon's refused or failed tables plus its pass-level problems, one entry per table or kind."""
137
+ report = report if isinstance(report, dict) else {}
138
+ tables = report.get("tables") if isinstance(report.get("tables"), dict) else {}
139
+ merged = {}
140
+
141
+ def add(key, label, code, cause):
142
+ entry = merged.setdefault((key, code), {"key": key, "label": label, "code": code, "causes": []})
143
+ if cause and cause not in entry["causes"]:
144
+ entry["causes"].append(cause)
145
+
146
+ covered = set()
147
+ for table, row in tables.items():
148
+ row = row if isinstance(row, dict) else {}
149
+ state = str(row.get("state") or "")
150
+ if state in AMAZON_FAILED:
151
+ covered.add(str(table))
152
+ # `reason` is the plain sentence; `diagnostic` carries raw exception text.
153
+ add(str(table), str(table), state, str(row.get("reason") or state))
154
+ for problem in report.get("problems") or []:
155
+ text = str(problem)
156
+ head = re.match(r"([A-Za-z0-9_]+):\s", text)
157
+ name = head.group(1) if head else None
158
+ if name in covered:
159
+ continue
160
+ if name in tables:
161
+ add(name, name, "problem", text[head.end():])
162
+ else:
163
+ add(SYNC_KEY, "Amazon sync", "problem", text)
164
+ return [{"key": e["key"], "label": e["label"], "code": e["code"],
165
+ "cause": scrub(" / ".join(e["causes"]) or e["code"])} for e in merged.values()]
166
+
167
+
168
+ def amazon_observed(report):
169
+ """The keys this Amazon pass proved healthy or failing; any other key keeps its alert."""
170
+ report = report if isinstance(report, dict) else {}
171
+ tables = report.get("tables") if isinstance(report.get("tables"), dict) else {}
172
+ observed = {SYNC_KEY}
173
+ for table, row in tables.items():
174
+ state = str((row if isinstance(row, dict) else {}).get("state") or "done")
175
+ if state in AMAZON_GOOD or state in AMAZON_FAILED:
176
+ observed.add(str(table))
177
+ return observed
178
+
179
+
180
+ # ---- state ----------------------------------------------------------------------------------
181
+
182
+ def _alert_id(tenant, connector, key, code):
183
+ return json.dumps([tenant, connector, key, code])
184
+
185
+
186
+ def _scope(tenant, connector):
187
+ return json.dumps([tenant, connector])
188
+
189
+
190
+ def _due(alert, now):
191
+ """The dedupe rule: what one alert needs this pass, or None."""
192
+ if not alert.get("sent_at"):
193
+ return "failed"
194
+ last = _parse(alert.get("reminded_at") or alert.get("sent_at"))
195
+ if last is None or now - last >= REMIND_AFTER:
196
+ return "reminder"
197
+ return None
198
+
199
+
200
+ def _reconcile(document, tenant, connector, current, observed, now, claim):
201
+ """Fold this pass into the document; return the plan of what this pass must send."""
202
+ document = document if isinstance(document, dict) else {}
203
+ alerts = document.setdefault("alerts", {})
204
+ recovered = document.setdefault("recovered", {})
205
+ scopes = document.setdefault("scopes", {})
206
+ leases = document.setdefault("leases", {})
207
+ scope = _scope(tenant, connector)
208
+ state = scopes.setdefault(scope, {})
209
+ ok = state.setdefault("ok", {})
210
+ last_full = state.get("last_full") or {}
211
+ stamp = now.isoformat()
212
+ failing_keys = {f["key"] for f in current.values()}
213
+
214
+ for alert_id, failure in current.items():
215
+ alert = alerts.get(alert_id)
216
+ if alert is None and alert_id in recovered:
217
+ # Failing again before its RECOVERED went out: the owner never heard it cleared.
218
+ alert = {k: v for k, v in recovered.pop(alert_id).items() if k != "cleared_at"}
219
+ if alert is None:
220
+ last_good = failure["last_good"]
221
+ if not last_good:
222
+ candidates = [ok.get(failure["key"])]
223
+ if last_full.get("at") and failure["key"] not in (last_full.get("failing") or []):
224
+ candidates.append(last_full["at"])
225
+ candidates = [c for c in candidates if _parse(c)]
226
+ last_good = max(candidates, key=_parse) if candidates else None
227
+ alert = {"tenant": tenant, "connector": connector, "first": stamp, "last_good": last_good,
228
+ "sent_at": None, "reminded_at": None}
229
+ alert.update(key=failure["key"], label=failure["label"], code=failure["code"],
230
+ cause=failure["cause"])
231
+ alerts[alert_id] = alert
232
+
233
+ for alert_id in [a for a, v in alerts.items()
234
+ if v.get("tenant") == tenant and v.get("connector") == connector and a not in current]:
235
+ key = alerts[alert_id].get("key")
236
+ if observed is not None and key not in observed:
237
+ continue
238
+ alert = alerts.pop(alert_id)
239
+ ok[key] = stamp
240
+ if alert.get("sent_at"):
241
+ recovered[alert_id] = {**alert, "cleared_at": stamp}
242
+
243
+ if observed is None:
244
+ state["last_full"] = {"at": stamp, "failing": sorted(failing_keys)}
245
+ else:
246
+ for key in observed:
247
+ if key not in failing_keys:
248
+ ok[key] = stamp
249
+ if len(ok) > _OK_CAP:
250
+ state["ok"] = dict(sorted(ok.items(), key=lambda item: item[1])[-_OK_CAP:])
251
+
252
+ lease = leases.get(scope) or {}
253
+ held = lease.get("claim") not in (None, claim) and (_parse(lease.get("until")) or now) > now
254
+ if any(_pending(document, tenant, connector, now).values()) and not held:
255
+ leases[scope] = {"claim": claim, "until": (now + LEASE).isoformat()}
256
+ return document
257
+
258
+
259
+ def _pending(document, tenant, connector, now):
260
+ """What this tenant and connector still owes the owner: unsent FAILED, due reminders, unsent RECOVERED."""
261
+ plan = {"failed": [], "reminder": [], "recovered": []}
262
+ for alert_id, alert in sorted((document.get("alerts") or {}).items()):
263
+ if alert.get("tenant") == tenant and alert.get("connector") == connector:
264
+ need = _due(alert, now)
265
+ if need:
266
+ plan[need].append(alert_id)
267
+ plan["recovered"] = sorted(a for a, v in (document.get("recovered") or {}).items()
268
+ if v.get("tenant") == tenant and v.get("connector") == connector)
269
+ return plan
270
+
271
+
272
+ def _plan_of(document, tenant, connector, now, claim):
273
+ lease = (document.get("leases") or {}).get(_scope(tenant, connector)) or {}
274
+ if lease.get("claim") != claim:
275
+ return None
276
+ return _pending(document, tenant, connector, now)
277
+
278
+
279
+ def _item(alert):
280
+ tail = (f"last good {when(alert['last_good'])}" if alert.get("last_good")
281
+ else f"failing since {when(alert.get('first'))}")
282
+ return f"{alert['label']} ({alert['cause']}, {tail})"
283
+
284
+
285
+ def environment():
286
+ """LIVE, STAGING or LOCAL, from the Space id HF sets, so a staging alert never reads as live."""
287
+ space = (os.environ.get("SPACE_ID") or os.environ.get("APP_BASE_URL") or "").lower()
288
+ if "runloopable" in space:
289
+ return "LIVE"
290
+ return "STAGING" if "loopable" in space else "LOCAL"
291
+
292
+
293
+ def compose(tenant, connector, plan, document):
294
+ """One line for the whole pass: FAILED, STILL FAILING and RECOVERED, each listed once."""
295
+ alerts, recovered = document.get("alerts") or {}, document.get("recovered") or {}
296
+ where = f"{environment()} {tenant} {connector}"
297
+ parts = []
298
+ for kind, title in (("failed", "FAILED"), ("reminder", "STILL FAILING")):
299
+ items = [_item(alerts[a]) for a in sorted(plan[kind]) if a in alerts]
300
+ if items:
301
+ parts.append(f"Loopable sync {title}: {where}: " + ", ".join(items) + ".")
302
+ names = [recovered[a]["label"] for a in plan["recovered"] if a in recovered]
303
+ if names:
304
+ parts.append(f"Loopable sync RECOVERED: {where}: " + ", ".join(names) + ".")
305
+ text = " ".join(parts)
306
+ return text if len(text) <= MESSAGE_CAP else text[:MESSAGE_CAP - 3].rstrip() + "..."
307
+
308
+
309
+ def _mark(document, tenant, connector, plan, now, claim, delivered):
310
+ document = document if isinstance(document, dict) else {}
311
+ alerts = document.setdefault("alerts", {})
312
+ recovered = document.setdefault("recovered", {})
313
+ leases = document.setdefault("leases", {})
314
+ scope = _scope(tenant, connector)
315
+ if (leases.get(scope) or {}).get("claim") == claim:
316
+ leases.pop(scope, None)
317
+ if delivered:
318
+ stamp = now.isoformat()
319
+ for alert_id in plan["failed"]:
320
+ if alert_id in alerts:
321
+ alerts[alert_id].update(sent_at=stamp, reminded_at=stamp)
322
+ for alert_id in plan["reminder"]:
323
+ if alert_id in alerts:
324
+ alerts[alert_id]["reminded_at"] = stamp
325
+ for alert_id in plan["recovered"]:
326
+ recovered.pop(alert_id, None)
327
+ return document
328
+
329
+
330
+ # ---- entry point ----------------------------------------------------------------------------
331
+
332
+ def notify(tenant, connector, failures, *, observed=None, now=None, sender=None, store=None):
333
+ """Record this pass's failures and send at most one Telegram line; never raises.
334
+
335
+ `failures` is the full current list for the tenant and connector; [] means healthy.
336
+ `observed` limits clearing to the keys this pass checked; None means the list is complete.
337
+ """
338
+ result = {"ok": False, "sent": False, "failed": [], "reminder": [], "recovered": [], "cause": None}
339
+ try:
340
+ now = (now or utc_now()).astimezone(timezone.utc)
341
+ tenant, connector = str(tenant), str(connector)
342
+ store = store if store is not None else store_backend()
343
+ current = {}
344
+ for failure in failures or []:
345
+ clean = _clean(failure)
346
+ if clean:
347
+ current[_alert_id(tenant, connector, clean["key"], clean["code"])] = clean
348
+ observed = None if observed is None else {str(key) for key in observed}
349
+ claim = uuid4().hex
350
+ # Save first, so a delivery marker from a past pass is never re-sent.
351
+ document = store.update(DOCUMENT, lambda doc: _reconcile(
352
+ doc, tenant, connector, current, observed, now, claim))
353
+ plan = _plan_of(document or {}, tenant, connector, now, claim)
354
+ if plan is None or not any(plan.values()):
355
+ result.update(ok=True, cause=None if plan is not None else "another pass is sending")
356
+ return result
357
+ text = compose(tenant, connector, plan, document)
358
+ try:
359
+ response = (sender or default_sender)(text)
360
+ delivered, detail = (response if isinstance(response, tuple) and len(response) == 2
361
+ else (response is not False, None))
362
+ except Exception: # noqa: BLE001
363
+ delivered, detail = False, "Telegram alert could not be sent."
364
+ delivered = bool(delivered)
365
+ store.update(DOCUMENT, lambda doc: _mark(doc, tenant, connector, plan, now, claim, delivered))
366
+ result.update({kind: list(ids) for kind, ids in plan.items()})
367
+ result.update(ok=delivered, sent=delivered,
368
+ cause=None if delivered else scrub(detail or "Telegram alert was not delivered."))
369
+ return result
370
+ except Exception as exc: # noqa: BLE001
371
+ result["cause"] = f"sync alert skipped ({type(exc).__name__})"
372
+ return result
web/public/AIOS_DEPLOY_RECEIPT CHANGED
@@ -1 +1 @@
1
- 933e8dbcdeac5d874c4e75c6e41071e0cb91231d4e925ee7850b24c5940d2ab5
 
1
+ f8fbaa9615aa924d504ee1421880fa42358ace44cae3257345d88081cc31bacb
web/src/connectors/OdooConfig.tsx CHANGED
@@ -92,7 +92,7 @@ export interface OdooFreshness {
92
  failure?: { code?: string; message?: string; at?: string | null } | null;
93
  /** D-2679: databases that failed while the rest of the sync succeeded. */
94
  failedDatabases?: { key: string; label?: string | null; code?: string | null; message?: string | null;
95
- stage?: string | null; at?: string | null }[];
96
  syncAt?: string | null;
97
  syncTz?: string | null;
98
  nextSyncAt?: string | null;
@@ -149,7 +149,9 @@ export function freshnessWarnings(f: OdooFreshness | null | undefined): string[]
149
  for (const db of Array.isArray(f.failedDatabases) ? f.failedDatabases : []) {
150
  const label = (db?.label || db?.key || "").trim();
151
  const message = (db?.message ?? "").trim();
152
- if (label) lines.push(`${label}${FAILED_DATABASE_LINE}${message ? ` ${message}` : ""}`);
 
 
153
  }
154
  return lines;
155
  }
 
92
  failure?: { code?: string; message?: string; at?: string | null } | null;
93
  /** D-2679: databases that failed while the rest of the sync succeeded. */
94
  failedDatabases?: { key: string; label?: string | null; code?: string | null; message?: string | null;
95
+ stage?: string | null; at?: string | null; cause?: string | null }[];
96
  syncAt?: string | null;
97
  syncTz?: string | null;
98
  nextSyncAt?: string | null;
 
149
  for (const db of Array.isArray(f.failedDatabases) ? f.failedDatabases : []) {
150
  const label = (db?.label || db?.key || "").trim();
151
  const message = (db?.message ?? "").trim();
152
+ // The cause stays on the same line: failedDatabasesStatus reads every line as a database.
153
+ const cause = (db?.cause ?? "").trim();
154
+ if (label) lines.push(`${label}${FAILED_DATABASE_LINE}${message ? ` ${message}` : ""}${cause ? ` Cause: ${cause}` : ""}`);
155
  }
156
  return lines;
157
  }