Spaces:
Running on CPU Upgrade
Running on CPU Upgrade
File size: 9,089 Bytes
4a80e55 971c29d 4a80e55 89af3b2 4a80e55 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 | """One finite AIOS automation job for ECS/Fargate.
The container has no loop and no service. It claims one durable Postgres job, holds a tenant +
automation advisory lock, performs only the requested work, reconciles follow-up schedules, and
exits. At-least-once delivery is therefore safe and idle AWS compute is literally zero tasks.
"""
from __future__ import annotations
import datetime as dt
import hashlib
import json
import os
import sys
import traceback
def _load_job() -> dict:
raw = str(os.environ.get("AIOS_AUTOMATION_JOB") or "").strip()
if not raw:
raise RuntimeError("AIOS_AUTOMATION_JOB is required")
job = json.loads(raw)
if not isinstance(job, dict):
raise RuntimeError("AIOS_AUTOMATION_JOB must be an object")
tenant = str(job.get("tenant") or "").strip().lower()
mode = str(job.get("mode") or "").strip().lower()
if not tenant or mode not in {"schedule", "email", "pending", "manual", "event", "probe"}:
raise RuntimeError("job tenant/mode is invalid")
job["tenant"], job["mode"] = tenant, mode
job["automationId"] = str(job.get("automationId") or "")
return job
def _job_id(job: dict) -> str:
supplied = str(job.get("requestId") or job.get("jobId") or "").strip()
if supplied:
return supplied[:180]
identity = "\0".join(str(job.get(k) or "") for k in
("tenant", "automationId", "mode", "generation", "scheduledTime"))
return "scheduled-" + hashlib.sha256(identity.encode()).hexdigest()
def _scheduled_for(job: dict):
raw = str(job.get("scheduledTime") or "").strip()
if not raw or raw.startswith("<aws.scheduler."):
return None
try:
return dt.datetime.fromisoformat(raw.replace("Z", "+00:00"))
except ValueError:
return None
def _finish(con, job_id: str, status: str, detail: dict):
with con.cursor() as cur:
cur.execute("""
UPDATE control.automation_jobs
SET status=%s, finished_at=now(), detail=detail || %s::jsonb
WHERE job_id=%s
""", (status, json.dumps(detail, default=str), job_id))
con.commit()
def _claim(job: dict):
import psycopg
from core import store_pg
store_pg.check_write(job["tenant"], operation="automation worker job claim")
url = str(os.environ.get("DATABASE_URL") or "").strip()
if not url:
raise RuntimeError("DATABASE_URL is required")
ca = os.environ.get("AIOS_DATABASE_CA_PEM_B64")
options = {"sslrootcert": store_pg._database_ca_file(ca)} if ca is not None else {}
con = psycopg.connect(url, **options)
lock_name = f"aios-automation:{job['tenant']}:{job['automationId'] or job['mode']}"
with con.cursor() as cur:
cur.execute("SELECT pg_try_advisory_lock(hashtextextended(%s, 0))", (lock_name,))
if not cur.fetchone()[0]:
con.close()
return None, "concurrent"
jid = _job_id(job)
cur.execute("""
INSERT INTO control.automation_jobs
(job_id, tenant_slug, automation_id, trigger_kind, generation, scheduled_for,
status, attempts, started_at, detail)
VALUES (%s,%s,%s,%s,%s,%s,'running',1,now(),'{}'::jsonb)
ON CONFLICT (job_id) DO NOTHING
""", (jid, job["tenant"], job["automationId"], job["mode"],
str(job.get("generation") or ""), _scheduled_for(job)))
inserted = cur.rowcount == 1
cur.execute("SELECT status, attempts, detail FROM control.automation_jobs WHERE job_id=%s",
(jid,))
status, attempts, detail = cur.fetchone()
if status in ("completed", "skipped"):
con.commit()
con.close()
return None, status
cur.execute("""
UPDATE control.automation_jobs
SET status='running', attempts=%s, started_at=now(), finished_at=NULL
WHERE job_id=%s
""", (max(1, int(attempts) + (0 if inserted else 1)), jid))
con.commit()
return con, dict(detail or {})
def _definition_refusal(job: dict, defn: dict | None, engine, control) -> str:
mode = job["mode"]
if mode == "probe":
return ""
if not defn:
return "automation no longer exists"
expected = str(job.get("generation") or "")
if expected and expected != control.generation(defn):
return "stale schedule generation"
if mode == "schedule" and not (defn.get("schedule") or {}).get("enabled"):
return "schedule is disabled"
trigger = defn.get("trigger") or {}
if mode == "email" and not (trigger.get("key") == "email"
and trigger.get("enabled", True)
and not trigger.get("paused")
and trigger.get("configured", True)):
return "email trigger is disabled or stale"
if mode == "pending" and str(defn.get("id")) not in engine.pending_collect_ids(
None, definitions={str(defn.get("id")): defn}):
return "paid vendor result is no longer pending"
return ""
def _run_business(job: dict, rt, defn: dict, engine):
mode = job["mode"]
auto_id = job["automationId"]
if mode == "probe":
return {"probe": "ok"}, False
if mode == "email":
result = engine.email_poll(rt, job["tenant"], auto_id, defn)
return {"email": "fired" if result is not None else "idle"}, result is not None
entry = engine.run_now(rt, job["tenant"], auto_id,
username=str(job.get("username") or "automation"),
rows=list(job.get("rows") or [])[:100] or None)
if entry is None:
return {"run": "not-started"}, False
return {"run": entry}, True
def main() -> int:
job = _load_job()
os.environ["AIOS_TENANT"] = job["tenant"]
os.environ["AIOS_AUTOMATION_INLINE"] = "1"
os.environ.setdefault("STORE_BACKEND", "pg")
os.environ.setdefault("SAFE_MODE", "1")
# Imports happen after tenant/identity env is final. Nothing imports FastAPI or starts an
# application server in this worker image.
import automation_control as control
import automation_engine as engine
import assistant_feed # Register the notice sink before isolated engine execution.
from harness import runtime
con, prior = _claim(job)
if con is None:
print(json.dumps({"jobId": _job_id(job), "result": prior}, sort_keys=True))
return 0
jid = _job_id(job)
try:
rt = runtime.get_runtime(job["tenant"])
defs = engine.all_definitions(rt)
defn = defs.get(job["automationId"])
refusal = _definition_refusal(job, defn, engine, control)
if refusal:
_finish(con, jid, "skipped", {"reason": refusal})
print(json.dumps({"jobId": jid, "result": "skipped", "reason": refusal},
sort_keys=True))
return 0
detail = dict(prior or {})
did_work = bool(detail.get("businessComplete"))
if not detail.get("businessComplete"):
outcome, did_work = _run_business(job, rt, defn, engine)
detail.update(outcome)
detail["businessComplete"] = True
with con.cursor() as cur:
cur.execute("UPDATE control.automation_jobs SET detail=%s::jsonb WHERE job_id=%s",
(json.dumps(detail, default=str), jid))
con.commit()
# Derived cells are refreshed only after a real run, never on an idle email check and
# never because a clock woke up. These functions persist only when values changed.
if did_work and not detail.get("derivedComplete"):
engine.refresh_relations(rt)
engine._refresh_source_rollups(rt, job["tenant"])
detail["derivedComplete"] = True
with con.cursor() as cur:
cur.execute("UPDATE control.automation_jobs SET detail=%s::jsonb WHERE job_id=%s",
(json.dumps(detail, default=str), jid))
con.commit()
if job["mode"] != "probe":
current = engine.all_definitions(rt).get(job["automationId"])
if current:
control.sync_definition(job["tenant"], current)
control.sync_pending(job["tenant"], current)
else:
control.delete_definition(job["tenant"], job["automationId"])
_finish(con, jid, "completed", {"controlSynced": True})
print(json.dumps({"jobId": jid, "result": "completed", "didWork": did_work},
sort_keys=True))
return 0
except Exception as exc:
try:
_finish(con, jid, "failed", {"error": type(exc).__name__,
"message": str(exc)[:500]})
except Exception:
pass
traceback.print_exc()
return 1
finally:
try:
con.close()
except Exception:
pass
if __name__ == "__main__":
raise SystemExit(main())
|