File size: 9,089 Bytes
5a07a58
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4325491
 
 
5a07a58
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
8145bee
5a07a58
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
"""One finite AIOS automation job for ECS/Fargate.

The container has no loop and no service.  It claims one durable Postgres job, holds a tenant +
automation advisory lock, performs only the requested work, reconciles follow-up schedules, and
exits.  At-least-once delivery is therefore safe and idle AWS compute is literally zero tasks.
"""
from __future__ import annotations

import datetime as dt
import hashlib
import json
import os
import sys
import traceback


def _load_job() -> dict:
    raw = str(os.environ.get("AIOS_AUTOMATION_JOB") or "").strip()
    if not raw:
        raise RuntimeError("AIOS_AUTOMATION_JOB is required")
    job = json.loads(raw)
    if not isinstance(job, dict):
        raise RuntimeError("AIOS_AUTOMATION_JOB must be an object")
    tenant = str(job.get("tenant") or "").strip().lower()
    mode = str(job.get("mode") or "").strip().lower()
    if not tenant or mode not in {"schedule", "email", "pending", "manual", "event", "probe"}:
        raise RuntimeError("job tenant/mode is invalid")
    job["tenant"], job["mode"] = tenant, mode
    job["automationId"] = str(job.get("automationId") or "")
    return job


def _job_id(job: dict) -> str:
    supplied = str(job.get("requestId") or job.get("jobId") or "").strip()
    if supplied:
        return supplied[:180]
    identity = "\0".join(str(job.get(k) or "") for k in
                          ("tenant", "automationId", "mode", "generation", "scheduledTime"))
    return "scheduled-" + hashlib.sha256(identity.encode()).hexdigest()


def _scheduled_for(job: dict):
    raw = str(job.get("scheduledTime") or "").strip()
    if not raw or raw.startswith("<aws.scheduler."):
        return None
    try:
        return dt.datetime.fromisoformat(raw.replace("Z", "+00:00"))
    except ValueError:
        return None


def _finish(con, job_id: str, status: str, detail: dict):
    with con.cursor() as cur:
        cur.execute("""
            UPDATE control.automation_jobs
               SET status=%s, finished_at=now(), detail=detail || %s::jsonb
             WHERE job_id=%s
        """, (status, json.dumps(detail, default=str), job_id))
    con.commit()


def _claim(job: dict):
    import psycopg
    from core import store_pg

    store_pg.check_write(job["tenant"], operation="automation worker job claim")
    url = str(os.environ.get("DATABASE_URL") or "").strip()
    if not url:
        raise RuntimeError("DATABASE_URL is required")
    ca = os.environ.get("AIOS_DATABASE_CA_PEM_B64")
    options = {"sslrootcert": store_pg._database_ca_file(ca)} if ca is not None else {}
    con = psycopg.connect(url, **options)
    lock_name = f"aios-automation:{job['tenant']}:{job['automationId'] or job['mode']}"
    with con.cursor() as cur:
        cur.execute("SELECT pg_try_advisory_lock(hashtextextended(%s, 0))", (lock_name,))
        if not cur.fetchone()[0]:
            con.close()
            return None, "concurrent"
        jid = _job_id(job)
        cur.execute("""
            INSERT INTO control.automation_jobs
                (job_id, tenant_slug, automation_id, trigger_kind, generation, scheduled_for,
                 status, attempts, started_at, detail)
            VALUES (%s,%s,%s,%s,%s,%s,'running',1,now(),'{}'::jsonb)
            ON CONFLICT (job_id) DO NOTHING
        """, (jid, job["tenant"], job["automationId"], job["mode"],
              str(job.get("generation") or ""), _scheduled_for(job)))
        inserted = cur.rowcount == 1
        cur.execute("SELECT status, attempts, detail FROM control.automation_jobs WHERE job_id=%s",
                    (jid,))
        status, attempts, detail = cur.fetchone()
        if status in ("completed", "skipped"):
            con.commit()
            con.close()
            return None, status
        cur.execute("""
            UPDATE control.automation_jobs
               SET status='running', attempts=%s, started_at=now(), finished_at=NULL
             WHERE job_id=%s
        """, (max(1, int(attempts) + (0 if inserted else 1)), jid))
    con.commit()
    return con, dict(detail or {})


def _definition_refusal(job: dict, defn: dict | None, engine, control) -> str:
    mode = job["mode"]
    if mode == "probe":
        return ""
    if not defn:
        return "automation no longer exists"
    expected = str(job.get("generation") or "")
    if expected and expected != control.generation(defn):
        return "stale schedule generation"
    if mode == "schedule" and not (defn.get("schedule") or {}).get("enabled"):
        return "schedule is disabled"
    trigger = defn.get("trigger") or {}
    if mode == "email" and not (trigger.get("key") == "email"
                                and trigger.get("enabled", True)
                                and not trigger.get("paused")
                                and trigger.get("configured", True)):
        return "email trigger is disabled or stale"
    if mode == "pending" and str(defn.get("id")) not in engine.pending_collect_ids(
            None, definitions={str(defn.get("id")): defn}):
        return "paid vendor result is no longer pending"
    return ""


def _run_business(job: dict, rt, defn: dict, engine):
    mode = job["mode"]
    auto_id = job["automationId"]
    if mode == "probe":
        return {"probe": "ok"}, False
    if mode == "email":
        result = engine.email_poll(rt, job["tenant"], auto_id, defn)
        return {"email": "fired" if result is not None else "idle"}, result is not None
    entry = engine.run_now(rt, job["tenant"], auto_id,
                           username=str(job.get("username") or "automation"),
                           rows=list(job.get("rows") or [])[:100] or None)
    if entry is None:
        return {"run": "not-started"}, False
    return {"run": entry}, True


def main() -> int:
    job = _load_job()
    os.environ["AIOS_TENANT"] = job["tenant"]
    os.environ["AIOS_AUTOMATION_INLINE"] = "1"
    os.environ.setdefault("STORE_BACKEND", "pg")
    os.environ.setdefault("SAFE_MODE", "1")

    # Imports happen after tenant/identity env is final.  Nothing imports FastAPI or starts an
    # application server in this worker image.
    import automation_control as control
    import automation_engine as engine
    import assistant_feed  # Register the notice sink before isolated engine execution.
    from harness import runtime

    con, prior = _claim(job)
    if con is None:
        print(json.dumps({"jobId": _job_id(job), "result": prior}, sort_keys=True))
        return 0
    jid = _job_id(job)
    try:
        rt = runtime.get_runtime(job["tenant"])
        defs = engine.all_definitions(rt)
        defn = defs.get(job["automationId"])
        refusal = _definition_refusal(job, defn, engine, control)
        if refusal:
            _finish(con, jid, "skipped", {"reason": refusal})
            print(json.dumps({"jobId": jid, "result": "skipped", "reason": refusal},
                             sort_keys=True))
            return 0

        detail = dict(prior or {})
        did_work = bool(detail.get("businessComplete"))
        if not detail.get("businessComplete"):
            outcome, did_work = _run_business(job, rt, defn, engine)
            detail.update(outcome)
            detail["businessComplete"] = True
            with con.cursor() as cur:
                cur.execute("UPDATE control.automation_jobs SET detail=%s::jsonb WHERE job_id=%s",
                            (json.dumps(detail, default=str), jid))
            con.commit()

        # Derived cells are refreshed only after a real run, never on an idle email check and
        # never because a clock woke up.  These functions persist only when values changed.
        if did_work and not detail.get("derivedComplete"):
            engine.refresh_relations(rt)
            engine._refresh_source_rollups(rt, job["tenant"])
            detail["derivedComplete"] = True
            with con.cursor() as cur:
                cur.execute("UPDATE control.automation_jobs SET detail=%s::jsonb WHERE job_id=%s",
                            (json.dumps(detail, default=str), jid))
            con.commit()

        if job["mode"] != "probe":
            current = engine.all_definitions(rt).get(job["automationId"])
            if current:
                control.sync_definition(job["tenant"], current)
                control.sync_pending(job["tenant"], current)
            else:
                control.delete_definition(job["tenant"], job["automationId"])
        _finish(con, jid, "completed", {"controlSynced": True})
        print(json.dumps({"jobId": jid, "result": "completed", "didWork": did_work},
                         sort_keys=True))
        return 0
    except Exception as exc:
        try:
            _finish(con, jid, "failed", {"error": type(exc).__name__,
                                         "message": str(exc)[:500]})
        except Exception:
            pass
        traceback.print_exc()
        return 1
    finally:
        try:
            con.close()
        except Exception:
            pass


if __name__ == "__main__":
    raise SystemExit(main())