loopable / api /automation_worker.py
fsanyoto's picture
Deploy AIOS web (React glide grid + FastAPI slice)
8145bee verified
Raw History Blame Contribute Delete
9.09 kB
"""One finite AIOS automation job for ECS/Fargate.
The container has no loop and no service. It claims one durable Postgres job, holds a tenant +
automation advisory lock, performs only the requested work, reconciles follow-up schedules, and
exits. At-least-once delivery is therefore safe and idle AWS compute is literally zero tasks.
"""
from __future__ import annotations
import datetime as dt
import hashlib
import json
import os
import sys
import traceback
def _load_job() -> dict:
raw = str(os.environ.get("AIOS_AUTOMATION_JOB") or "").strip()
if not raw:
raise RuntimeError("AIOS_AUTOMATION_JOB is required")
job = json.loads(raw)
if not isinstance(job, dict):
raise RuntimeError("AIOS_AUTOMATION_JOB must be an object")
tenant = str(job.get("tenant") or "").strip().lower()
mode = str(job.get("mode") or "").strip().lower()
if not tenant or mode not in {"schedule", "email", "pending", "manual", "event", "probe"}:
raise RuntimeError("job tenant/mode is invalid")
job["tenant"], job["mode"] = tenant, mode
job["automationId"] = str(job.get("automationId") or "")
return job
def _job_id(job: dict) -> str:
supplied = str(job.get("requestId") or job.get("jobId") or "").strip()
if supplied:
return supplied[:180]
identity = "\0".join(str(job.get(k) or "") for k in
("tenant", "automationId", "mode", "generation", "scheduledTime"))
return "scheduled-" + hashlib.sha256(identity.encode()).hexdigest()
def _scheduled_for(job: dict):
raw = str(job.get("scheduledTime") or "").strip()
if not raw or raw.startswith("<aws.scheduler."):
return None
try:
return dt.datetime.fromisoformat(raw.replace("Z", "+00:00"))
except ValueError:
return None
def _finish(con, job_id: str, status: str, detail: dict):
with con.cursor() as cur:
cur.execute("""
UPDATE control.automation_jobs
SET status=%s, finished_at=now(), detail=detail || %s::jsonb
WHERE job_id=%s
""", (status, json.dumps(detail, default=str), job_id))
con.commit()
def _claim(job: dict):
import psycopg
from core import store_pg
store_pg.check_write(job["tenant"], operation="automation worker job claim")
url = str(os.environ.get("DATABASE_URL") or "").strip()
if not url:
raise RuntimeError("DATABASE_URL is required")
ca = os.environ.get("AIOS_DATABASE_CA_PEM_B64")
options = {"sslrootcert": store_pg._database_ca_file(ca)} if ca is not None else {}
con = psycopg.connect(url, **options)
lock_name = f"aios-automation:{job['tenant']}:{job['automationId'] or job['mode']}"
with con.cursor() as cur:
cur.execute("SELECT pg_try_advisory_lock(hashtextextended(%s, 0))", (lock_name,))
if not cur.fetchone()[0]:
con.close()
return None, "concurrent"
jid = _job_id(job)
cur.execute("""
INSERT INTO control.automation_jobs
(job_id, tenant_slug, automation_id, trigger_kind, generation, scheduled_for,
status, attempts, started_at, detail)
VALUES (%s,%s,%s,%s,%s,%s,'running',1,now(),'{}'::jsonb)
ON CONFLICT (job_id) DO NOTHING
""", (jid, job["tenant"], job["automationId"], job["mode"],
str(job.get("generation") or ""), _scheduled_for(job)))
inserted = cur.rowcount == 1
cur.execute("SELECT status, attempts, detail FROM control.automation_jobs WHERE job_id=%s",
(jid,))
status, attempts, detail = cur.fetchone()
if status in ("completed", "skipped"):
con.commit()
con.close()
return None, status
cur.execute("""
UPDATE control.automation_jobs
SET status='running', attempts=%s, started_at=now(), finished_at=NULL
WHERE job_id=%s
""", (max(1, int(attempts) + (0 if inserted else 1)), jid))
con.commit()
return con, dict(detail or {})
def _definition_refusal(job: dict, defn: dict | None, engine, control) -> str:
mode = job["mode"]
if mode == "probe":
return ""
if not defn:
return "automation no longer exists"
expected = str(job.get("generation") or "")
if expected and expected != control.generation(defn):
return "stale schedule generation"
if mode == "schedule" and not (defn.get("schedule") or {}).get("enabled"):
return "schedule is disabled"
trigger = defn.get("trigger") or {}
if mode == "email" and not (trigger.get("key") == "email"
and trigger.get("enabled", True)
and not trigger.get("paused")
and trigger.get("configured", True)):
return "email trigger is disabled or stale"
if mode == "pending" and str(defn.get("id")) not in engine.pending_collect_ids(
None, definitions={str(defn.get("id")): defn}):
return "paid vendor result is no longer pending"
return ""
def _run_business(job: dict, rt, defn: dict, engine):
mode = job["mode"]
auto_id = job["automationId"]
if mode == "probe":
return {"probe": "ok"}, False
if mode == "email":
result = engine.email_poll(rt, job["tenant"], auto_id, defn)
return {"email": "fired" if result is not None else "idle"}, result is not None
entry = engine.run_now(rt, job["tenant"], auto_id,
username=str(job.get("username") or "automation"),
rows=list(job.get("rows") or [])[:100] or None)
if entry is None:
return {"run": "not-started"}, False
return {"run": entry}, True
def main() -> int:
job = _load_job()
os.environ["AIOS_TENANT"] = job["tenant"]
os.environ["AIOS_AUTOMATION_INLINE"] = "1"
os.environ.setdefault("STORE_BACKEND", "pg")
os.environ.setdefault("SAFE_MODE", "1")
# Imports happen after tenant/identity env is final. Nothing imports FastAPI or starts an
# application server in this worker image.
import automation_control as control
import automation_engine as engine
import assistant_feed # Register the notice sink before isolated engine execution.
from harness import runtime
con, prior = _claim(job)
if con is None:
print(json.dumps({"jobId": _job_id(job), "result": prior}, sort_keys=True))
return 0
jid = _job_id(job)
try:
rt = runtime.get_runtime(job["tenant"])
defs = engine.all_definitions(rt)
defn = defs.get(job["automationId"])
refusal = _definition_refusal(job, defn, engine, control)
if refusal:
_finish(con, jid, "skipped", {"reason": refusal})
print(json.dumps({"jobId": jid, "result": "skipped", "reason": refusal},
sort_keys=True))
return 0
detail = dict(prior or {})
did_work = bool(detail.get("businessComplete"))
if not detail.get("businessComplete"):
outcome, did_work = _run_business(job, rt, defn, engine)
detail.update(outcome)
detail["businessComplete"] = True
with con.cursor() as cur:
cur.execute("UPDATE control.automation_jobs SET detail=%s::jsonb WHERE job_id=%s",
(json.dumps(detail, default=str), jid))
con.commit()
# Derived cells are refreshed only after a real run, never on an idle email check and
# never because a clock woke up. These functions persist only when values changed.
if did_work and not detail.get("derivedComplete"):
engine.refresh_relations(rt)
engine._refresh_source_rollups(rt, job["tenant"])
detail["derivedComplete"] = True
with con.cursor() as cur:
cur.execute("UPDATE control.automation_jobs SET detail=%s::jsonb WHERE job_id=%s",
(json.dumps(detail, default=str), jid))
con.commit()
if job["mode"] != "probe":
current = engine.all_definitions(rt).get(job["automationId"])
if current:
control.sync_definition(job["tenant"], current)
control.sync_pending(job["tenant"], current)
else:
control.delete_definition(job["tenant"], job["automationId"])
_finish(con, jid, "completed", {"controlSynced": True})
print(json.dumps({"jobId": jid, "result": "completed", "didWork": did_work},
sort_keys=True))
return 0
except Exception as exc:
try:
_finish(con, jid, "failed", {"error": type(exc).__name__,
"message": str(exc)[:500]})
except Exception:
pass
traceback.print_exc()
return 1
finally:
try:
con.close()
except Exception:
pass
if __name__ == "__main__":
raise SystemExit(main())