Download api/automation_worker.py from fsanyoto/loopable: direct link, hf CLI and curl.
- Browser
- Download file 9.09 kB
-
https://huggingface.co/spaces/fsanyoto/loopable/resolve/main/api/automation_worker.py
- Command line
-
hf download hf://spaces/fsanyoto/loopable/api/automation_worker.py
-
curl -L -o automation_worker.py https://huggingface.co/spaces/fsanyoto/loopable/resolve/main/api/automation_worker.py
9.09 kB
| """One finite AIOS automation job for ECS/Fargate. | |
| The container has no loop and no service. It claims one durable Postgres job, holds a tenant + | |
| automation advisory lock, performs only the requested work, reconciles follow-up schedules, and | |
| exits. At-least-once delivery is therefore safe and idle AWS compute is literally zero tasks. | |
| """ | |
| from __future__ import annotations | |
| import datetime as dt | |
| import hashlib | |
| import json | |
| import os | |
| import sys | |
| import traceback | |
| def _load_job() -> dict: | |
| raw = str(os.environ.get("AIOS_AUTOMATION_JOB") or "").strip() | |
| if not raw: | |
| raise RuntimeError("AIOS_AUTOMATION_JOB is required") | |
| job = json.loads(raw) | |
| if not isinstance(job, dict): | |
| raise RuntimeError("AIOS_AUTOMATION_JOB must be an object") | |
| tenant = str(job.get("tenant") or "").strip().lower() | |
| mode = str(job.get("mode") or "").strip().lower() | |
| if not tenant or mode not in {"schedule", "email", "pending", "manual", "event", "probe"}: | |
| raise RuntimeError("job tenant/mode is invalid") | |
| job["tenant"], job["mode"] = tenant, mode | |
| job["automationId"] = str(job.get("automationId") or "") | |
| return job | |
| def _job_id(job: dict) -> str: | |
| supplied = str(job.get("requestId") or job.get("jobId") or "").strip() | |
| if supplied: | |
| return supplied[:180] | |
| identity = "\0".join(str(job.get(k) or "") for k in | |
| ("tenant", "automationId", "mode", "generation", "scheduledTime")) | |
| return "scheduled-" + hashlib.sha256(identity.encode()).hexdigest() | |
| def _scheduled_for(job: dict): | |
| raw = str(job.get("scheduledTime") or "").strip() | |
| if not raw or raw.startswith("<aws.scheduler."): | |
| return None | |
| try: | |
| return dt.datetime.fromisoformat(raw.replace("Z", "+00:00")) | |
| except ValueError: | |
| return None | |
| def _finish(con, job_id: str, status: str, detail: dict): | |
| with con.cursor() as cur: | |
| cur.execute(""" | |
| UPDATE control.automation_jobs | |
| SET status=%s, finished_at=now(), detail=detail || %s::jsonb | |
| WHERE job_id=%s | |
| """, (status, json.dumps(detail, default=str), job_id)) | |
| con.commit() | |
| def _claim(job: dict): | |
| import psycopg | |
| from core import store_pg | |
| store_pg.check_write(job["tenant"], operation="automation worker job claim") | |
| url = str(os.environ.get("DATABASE_URL") or "").strip() | |
| if not url: | |
| raise RuntimeError("DATABASE_URL is required") | |
| ca = os.environ.get("AIOS_DATABASE_CA_PEM_B64") | |
| options = {"sslrootcert": store_pg._database_ca_file(ca)} if ca is not None else {} | |
| con = psycopg.connect(url, **options) | |
| lock_name = f"aios-automation:{job['tenant']}:{job['automationId'] or job['mode']}" | |
| with con.cursor() as cur: | |
| cur.execute("SELECT pg_try_advisory_lock(hashtextextended(%s, 0))", (lock_name,)) | |
| if not cur.fetchone()[0]: | |
| con.close() | |
| return None, "concurrent" | |
| jid = _job_id(job) | |
| cur.execute(""" | |
| INSERT INTO control.automation_jobs | |
| (job_id, tenant_slug, automation_id, trigger_kind, generation, scheduled_for, | |
| status, attempts, started_at, detail) | |
| VALUES (%s,%s,%s,%s,%s,%s,'running',1,now(),'{}'::jsonb) | |
| ON CONFLICT (job_id) DO NOTHING | |
| """, (jid, job["tenant"], job["automationId"], job["mode"], | |
| str(job.get("generation") or ""), _scheduled_for(job))) | |
| inserted = cur.rowcount == 1 | |
| cur.execute("SELECT status, attempts, detail FROM control.automation_jobs WHERE job_id=%s", | |
| (jid,)) | |
| status, attempts, detail = cur.fetchone() | |
| if status in ("completed", "skipped"): | |
| con.commit() | |
| con.close() | |
| return None, status | |
| cur.execute(""" | |
| UPDATE control.automation_jobs | |
| SET status='running', attempts=%s, started_at=now(), finished_at=NULL | |
| WHERE job_id=%s | |
| """, (max(1, int(attempts) + (0 if inserted else 1)), jid)) | |
| con.commit() | |
| return con, dict(detail or {}) | |
| def _definition_refusal(job: dict, defn: dict | None, engine, control) -> str: | |
| mode = job["mode"] | |
| if mode == "probe": | |
| return "" | |
| if not defn: | |
| return "automation no longer exists" | |
| expected = str(job.get("generation") or "") | |
| if expected and expected != control.generation(defn): | |
| return "stale schedule generation" | |
| if mode == "schedule" and not (defn.get("schedule") or {}).get("enabled"): | |
| return "schedule is disabled" | |
| trigger = defn.get("trigger") or {} | |
| if mode == "email" and not (trigger.get("key") == "email" | |
| and trigger.get("enabled", True) | |
| and not trigger.get("paused") | |
| and trigger.get("configured", True)): | |
| return "email trigger is disabled or stale" | |
| if mode == "pending" and str(defn.get("id")) not in engine.pending_collect_ids( | |
| None, definitions={str(defn.get("id")): defn}): | |
| return "paid vendor result is no longer pending" | |
| return "" | |
| def _run_business(job: dict, rt, defn: dict, engine): | |
| mode = job["mode"] | |
| auto_id = job["automationId"] | |
| if mode == "probe": | |
| return {"probe": "ok"}, False | |
| if mode == "email": | |
| result = engine.email_poll(rt, job["tenant"], auto_id, defn) | |
| return {"email": "fired" if result is not None else "idle"}, result is not None | |
| entry = engine.run_now(rt, job["tenant"], auto_id, | |
| username=str(job.get("username") or "automation"), | |
| rows=list(job.get("rows") or [])[:100] or None) | |
| if entry is None: | |
| return {"run": "not-started"}, False | |
| return {"run": entry}, True | |
| def main() -> int: | |
| job = _load_job() | |
| os.environ["AIOS_TENANT"] = job["tenant"] | |
| os.environ["AIOS_AUTOMATION_INLINE"] = "1" | |
| os.environ.setdefault("STORE_BACKEND", "pg") | |
| os.environ.setdefault("SAFE_MODE", "1") | |
| # Imports happen after tenant/identity env is final. Nothing imports FastAPI or starts an | |
| # application server in this worker image. | |
| import automation_control as control | |
| import automation_engine as engine | |
| import assistant_feed # Register the notice sink before isolated engine execution. | |
| from harness import runtime | |
| con, prior = _claim(job) | |
| if con is None: | |
| print(json.dumps({"jobId": _job_id(job), "result": prior}, sort_keys=True)) | |
| return 0 | |
| jid = _job_id(job) | |
| try: | |
| rt = runtime.get_runtime(job["tenant"]) | |
| defs = engine.all_definitions(rt) | |
| defn = defs.get(job["automationId"]) | |
| refusal = _definition_refusal(job, defn, engine, control) | |
| if refusal: | |
| _finish(con, jid, "skipped", {"reason": refusal}) | |
| print(json.dumps({"jobId": jid, "result": "skipped", "reason": refusal}, | |
| sort_keys=True)) | |
| return 0 | |
| detail = dict(prior or {}) | |
| did_work = bool(detail.get("businessComplete")) | |
| if not detail.get("businessComplete"): | |
| outcome, did_work = _run_business(job, rt, defn, engine) | |
| detail.update(outcome) | |
| detail["businessComplete"] = True | |
| with con.cursor() as cur: | |
| cur.execute("UPDATE control.automation_jobs SET detail=%s::jsonb WHERE job_id=%s", | |
| (json.dumps(detail, default=str), jid)) | |
| con.commit() | |
| # Derived cells are refreshed only after a real run, never on an idle email check and | |
| # never because a clock woke up. These functions persist only when values changed. | |
| if did_work and not detail.get("derivedComplete"): | |
| engine.refresh_relations(rt) | |
| engine._refresh_source_rollups(rt, job["tenant"]) | |
| detail["derivedComplete"] = True | |
| with con.cursor() as cur: | |
| cur.execute("UPDATE control.automation_jobs SET detail=%s::jsonb WHERE job_id=%s", | |
| (json.dumps(detail, default=str), jid)) | |
| con.commit() | |
| if job["mode"] != "probe": | |
| current = engine.all_definitions(rt).get(job["automationId"]) | |
| if current: | |
| control.sync_definition(job["tenant"], current) | |
| control.sync_pending(job["tenant"], current) | |
| else: | |
| control.delete_definition(job["tenant"], job["automationId"]) | |
| _finish(con, jid, "completed", {"controlSynced": True}) | |
| print(json.dumps({"jobId": jid, "result": "completed", "didWork": did_work}, | |
| sort_keys=True)) | |
| return 0 | |
| except Exception as exc: | |
| try: | |
| _finish(con, jid, "failed", {"error": type(exc).__name__, | |
| "message": str(exc)[:500]}) | |
| except Exception: | |
| pass | |
| traceback.print_exc() | |
| return 1 | |
| finally: | |
| try: | |
| con.close() | |
| except Exception: | |
| pass | |
| if __name__ == "__main__": | |
| raise SystemExit(main()) | |