Deploy AIOS web (React glide grid + FastAPI slice)
Browse files- RELEASES.json +1 -1
- VERSION +1 -1
- api/automation_engine.py +160 -1
- platform/core/user_tables.py +20 -1
RELEASES.json
CHANGED
|
@@ -1,5 +1,5 @@
|
|
| 1 |
{
|
| 2 |
-
"current": "
|
| 3 |
"releases": [
|
| 4 |
{
|
| 5 |
"version": "v24",
|
|
|
|
| 1 |
{
|
| 2 |
+
"current": "b03615b",
|
| 3 |
"releases": [
|
| 4 |
{
|
| 5 |
"version": "v24",
|
VERSION
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
|
|
|
|
| 1 |
+
b03615b
|
api/automation_engine.py
CHANGED
|
@@ -3429,7 +3429,25 @@ def _ig_schema_contract(rt, profile_tables=None):
|
|
| 3429 |
continue
|
| 3430 |
label = str(table.get("label") or table_key)
|
| 3431 |
backlinks.append(_profile_backlink_field(table_key, label, bound))
|
| 3432 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3433 |
# `tracked` was the pre-Stage progress checkbox. Keeping both allows two progress states
|
| 3434 |
# to disagree, so it is retired by key just like the old nested Posts JSON.
|
| 3435 |
drops[table_key] = {"posts", "post_hashtags", "tracked"}
|
|
@@ -3985,6 +4003,126 @@ def backfill_post_snapshot_grain(rt, log=print):
|
|
| 3985 |
return len(changes)
|
| 3986 |
|
| 3987 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3988 |
def migrate_ig_tables(rt, log=print, only=""):
|
| 3989 |
"""Bring every Instagram Profile and canonical child table onto one current contract.
|
| 3990 |
|
|
@@ -4121,6 +4259,27 @@ def migrate_ig_tables(rt, log=print, only=""):
|
|
| 4121 |
# the stored old type is the idempotency key for unit-changing values such as engagement.
|
| 4122 |
wanted, drops = _ig_schema_contract(rt)
|
| 4123 |
stats["schema"] = int(_reconcile_ig_graph_fields(rt, wanted, drops))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4124 |
# ββ 2026-08-10 β THE TWO VALUE BACKFILLS, and they run AFTER the schema pass on purpose:
|
| 4125 |
# `backfill_post_snapshot_grain` writes into columns that pass has just declared, and running
|
| 4126 |
# it first would find nothing to fill and report a truthful zero about the wrong moment.
|
|
|
|
| 3429 |
continue
|
| 3430 |
label = str(table.get("label") or table_key)
|
| 3431 |
backlinks.append(_profile_backlink_field(table_key, label, bound))
|
| 3432 |
+
schema = _profile_schema_for(bound)
|
| 3433 |
+
# ββ 2026-08-10 (owner: *"retire the old hardcoded method and replace the Instagram
|
| 3434 |
+
# database work with correct Rollup and Link fields"*) β THE DERIVED OVERLAY, AND IT
|
| 3435 |
+
# BELONGS HERE RATHER THAN IN A SEPARATE MIGRATION.
|
| 3436 |
+
#
|
| 3437 |
+
# β A standalone converter would LOSE. `_reconcile_ig_graph_fields` overwrites every
|
| 3438 |
+
# contract key it owns β `rollup` included, by its own note β so a column converted beside
|
| 3439 |
+
# the contract is re-typed back to `int` on the next reconcile, silently, hours later.
|
| 3440 |
+
# Making the CONTRACT itself say "this column is derived" leaves exactly one writer of the
|
| 3441 |
+
# schema instead of two that disagree.
|
| 3442 |
+
#
|
| 3443 |
+
# β THE SET IS PER TENANT AND IS A PROOF, NOT A LIST. `_derived_profile_columns` converts a
|
| 3444 |
+
# column only where this tenant's own rows show the fold already equals the stored value β
|
| 3445 |
+
# measured, because a named list would have blanked 221 live cells on nurilab alone
|
| 3446 |
+
# (`avg_engagement` and `category`, 62 each). A tenant whose series is thinner converts
|
| 3447 |
+
# fewer columns and loses nothing; as its history fills in, later passes convert more.
|
| 3448 |
+
derived = set(_derived_profile_columns(table, tables.get(IG_SNAPSHOTS_TABLE) or {}))
|
| 3449 |
+
wanted[table_key] = [_as_derived_field(f) if str(f.get("key")) in derived else f
|
| 3450 |
+
for f in schema]
|
| 3451 |
# `tracked` was the pre-Stage progress checkbox. Keeping both allows two progress states
|
| 3452 |
# to disagree, so it is retired by key just like the old nested Posts JSON.
|
| 3453 |
drops[table_key] = {"posts", "post_hashtags", "tracked"}
|
|
|
|
| 4003 |
return len(changes)
|
| 4004 |
|
| 4005 |
|
| 4006 |
+
#: ββ 2026-08-10 (owner: *"retire the old hardcoded method and replace the Instagram database work
|
| 4007 |
+
#: with correct Rollup and Link fields"*) β THE PROFILE COLUMNS THAT MAY BECOME ROLLUPS.
|
| 4008 |
+
#:
|
| 4009 |
+
#: A column qualifies when the SERIES has somewhere to put it: `SNAPSHOT_FIELDS` carries the same
|
| 4010 |
+
#: key, so `latest` over `profile_snapshots_link` can re-derive it. That is 27 of the 44 preset
|
| 4011 |
+
#: columns β the other 17 are identity/bookkeeping (`handle`, `platform`, `enriched_at`), already
|
| 4012 |
+
#: relational (the 12 links and rollups), or have no observation behind them at all
|
| 4013 |
+
#: (`location_guess`/`location_confidence`, which WE infer rather than read).
|
| 4014 |
+
#:
|
| 4015 |
+
#: ββ AND "DERIVABLE" IS NOT "SAFE", WHICH IS THE WHOLE REASON THIS IS A PREDICATE AND NOT A LIST.
|
| 4016 |
+
#: MEASURED on nurilab before any conversion: a bare `latest` would BLANK 221 filled cells β
|
| 4017 |
+
#: `avg_engagement` and `category` lose 62 each β because the discovery reads that produced those
|
| 4018 |
+
#: values were never recorded as observations (the hole this same wave closed going forward, in its
|
| 4019 |
+
#: historical form: 62 profiles hold exactly ONE observation, the enrichment scrape, dated three
|
| 4020 |
+
#: days AFTER the discovery run whose numbers are on their row). A migration that converts on a
|
| 4021 |
+
#: NAMED LIST would do that damage on any tenant whose series is thinner than the list's author
|
| 4022 |
+
#: assumed.
|
| 4023 |
+
#:
|
| 4024 |
+
#: β So the migration converts a column only where it can PROVE, on this tenant's own rows, that
|
| 4025 |
+
#: the derived value equals the stored one everywhere. See `_convertible_profile_columns`.
|
| 4026 |
+
#: β Re-runnable by design: as the series fills in, later passes convert more. A column that is not
|
| 4027 |
+
#: safe today is not refused forever, it is simply not converted yet.
|
| 4028 |
+
IG_DERIVABLE_KEYS = tuple(
|
| 4029 |
+
fd["key"] for fd in PRESET_PROFILE_FIELDS
|
| 4030 |
+
if fd.get("type") not in ("link", "rollup")
|
| 4031 |
+
and fd["key"] not in ("platform", "handle", "enriched_at", "first_found", "last_found",
|
| 4032 |
+
"found_count", "created_by")
|
| 4033 |
+
and fd["key"] in {s["key"] for s in SNAPSHOT_FIELDS})
|
| 4034 |
+
|
| 4035 |
+
|
| 4036 |
+
def _derived_profile_bag(key):
|
| 4037 |
+
"""The rollup that replaces the mapper's write of ONE profile column.
|
| 4038 |
+
|
| 4039 |
+
β `where: is_not_empty` IS NOT DECORATION. A bare `latest` returns the newest observation's
|
| 4040 |
+
value INCLUDING ITS BLANK, so a cheap pull that did not read the field would erase what an
|
| 4041 |
+
expensive one learned β the exact asymmetry `upsert_rows`' merge exists to prevent on the
|
| 4042 |
+
scalar side. `where` runs BEFORE the ranking (contract C1), so this reads "the newest
|
| 4043 |
+
observation that ACTUALLY READ this field", which is what the materialised cell meant.
|
| 4044 |
+
MEASURED: it rescues 5 of the 12 columns a bare `latest` would damage; the other 7 need the
|
| 4045 |
+
observation itself, which is why the guard below is a proof and not a hope.
|
| 4046 |
+
"""
|
| 4047 |
+
return {"link": "profile_snapshots_link", "field": key, "fn": "latest",
|
| 4048 |
+
"sortBy": "pulled_at", "sortDir": "desc",
|
| 4049 |
+
"where": [{"field": key, "op": "is_not_empty"}], "whereConj": "and"}
|
| 4050 |
+
|
| 4051 |
+
|
| 4052 |
+
def _derived_profile_columns(table, snapshots):
|
| 4053 |
+
"""Which of `IG_DERIVABLE_KEYS` should be DECLARED as rollups on THIS table.
|
| 4054 |
+
|
| 4055 |
+
= the ones already converted, PLUS the ones whose derived value equals the stored value on
|
| 4056 |
+
every row. Returns keys.
|
| 4057 |
+
|
| 4058 |
+
ββ IT MUST RE-ASSERT THE ONES ALREADY CONVERTED, and forgetting that is a one-line revert with
|
| 4059 |
+
a several-hour fuse. `_reconcile_ig_graph_fields` overwrites every contract key it owns β
|
| 4060 |
+
`rollup` included, by design β so a converted column that this function stopped naming would be
|
| 4061 |
+
re-typed back to `int`/`text` on the next reconcile, its bag dropped, and its value re-written
|
| 4062 |
+
by the mapper. The edit would look applied and undo itself later, which is exactly
|
| 4063 |
+
[[preset-unlock-needs-a-custody-stamp]]'s shape. An already-converted column is therefore
|
| 4064 |
+
included unconditionally rather than re-proved: its cells ARE the derived values, so a proof
|
| 4065 |
+
would be comparing the fold against itself.
|
| 4066 |
+
|
| 4067 |
+
β THE PROOF RUNS PER TENANT, PER COLUMN, AT MIGRATION TIME β never against a list somebody
|
| 4068 |
+
measured on one workspace. `followers` derives exactly on nurilab and could be thin somewhere
|
| 4069 |
+
else; `category` is damaged on nurilab and might be perfect elsewhere. The only honest
|
| 4070 |
+
authority is the data in front of the migration.
|
| 4071 |
+
β A column with NO stored values anywhere converts too, and that is deliberate rather than an
|
| 4072 |
+
oversight: there is nothing to lose, and leaving it materialised would mean the schema differs
|
| 4073 |
+
between two tenants for no reason a reader could discover.
|
| 4074 |
+
"""
|
| 4075 |
+
fields = {str(f.get("key")): f for f in (table or {}).get("fields") or []}
|
| 4076 |
+
rows = (table or {}).get("rows") or {}
|
| 4077 |
+
by_subject = {}
|
| 4078 |
+
for snap in ((snapshots or {}).get("rows") or {}).values():
|
| 4079 |
+
key = str((snap or {}).get("influencer_key") or "").strip().lower()
|
| 4080 |
+
if key:
|
| 4081 |
+
by_subject.setdefault(key, []).append(snap or {})
|
| 4082 |
+
# The engine's own ordering, once per subject β newest first, unrankable rows last.
|
| 4083 |
+
ordered = {}
|
| 4084 |
+
for subject, observations in by_subject.items():
|
| 4085 |
+
keyed = [(o, _sort_key(o.get("pulled_at"), "date")) for o in observations]
|
| 4086 |
+
rankable = [(o, k) for o, k in keyed if k is not None]
|
| 4087 |
+
rankable.sort(key=lambda ok: ok[1], reverse=True)
|
| 4088 |
+
ordered[subject] = [o for o, _k in rankable] + [o for o, k in keyed if k is None]
|
| 4089 |
+
# β A rollup needs the LINK it folds. A profile table that has not been through the graph
|
| 4090 |
+
# reconcile has no `profile_snapshots_link`, and declaring one there would mint a column whose
|
| 4091 |
+
# link resolves to nothing β a blank cell wearing a configured column's clothes.
|
| 4092 |
+
if "profile_snapshots_link" not in fields:
|
| 4093 |
+
return []
|
| 4094 |
+
out = []
|
| 4095 |
+
for key in IG_DERIVABLE_KEYS:
|
| 4096 |
+
field = fields.get(key)
|
| 4097 |
+
if not field:
|
| 4098 |
+
continue
|
| 4099 |
+
if field.get("type") == "rollup":
|
| 4100 |
+
out.append(key) # already converted β see the note above on why
|
| 4101 |
+
continue
|
| 4102 |
+
declared = str(field.get("type") or "text")
|
| 4103 |
+
safe = True
|
| 4104 |
+
for row in rows.values():
|
| 4105 |
+
stored = str((row or {}).get(key) or "").strip()
|
| 4106 |
+
window = [o for o in ordered.get(
|
| 4107 |
+
str((row or {}).get("handle") or "").strip().lower(), [])
|
| 4108 |
+
if str(o.get(key) or "").strip() != ""] # the `where` guard, applied here
|
| 4109 |
+
derived = str(_rollup_fold("latest", [o.get(key) for o in window], declared)
|
| 4110 |
+
or "").strip()
|
| 4111 |
+
if stored != derived:
|
| 4112 |
+
safe = False
|
| 4113 |
+
break
|
| 4114 |
+
if safe:
|
| 4115 |
+
out.append(key)
|
| 4116 |
+
return out
|
| 4117 |
+
|
| 4118 |
+
|
| 4119 |
+
def _as_derived_field(field):
|
| 4120 |
+
"""One preset scalar declaration β the same column declared as a rollup over the series."""
|
| 4121 |
+
key = str(field.get("key") or "")
|
| 4122 |
+
return {**{k: v for k, v in field.items() if k not in ("agg",)},
|
| 4123 |
+
"type": "rollup", "rollup": _derived_profile_bag(key)}
|
| 4124 |
+
|
| 4125 |
+
|
| 4126 |
def migrate_ig_tables(rt, log=print, only=""):
|
| 4127 |
"""Bring every Instagram Profile and canonical child table onto one current contract.
|
| 4128 |
|
|
|
|
| 4259 |
# the stored old type is the idempotency key for unit-changing values such as engagement.
|
| 4260 |
wanted, drops = _ig_schema_contract(rt)
|
| 4261 |
stats["schema"] = int(_reconcile_ig_graph_fields(rt, wanted, drops))
|
| 4262 |
+
# ββ A SECOND CONTRACT PASS, AND IT IS NOT BELT-AND-BRACES β WITHOUT IT THIS FUNCTION IS NOT
|
| 4263 |
+
# IDEMPOTENT, which is the one property everything else here is built on.
|
| 4264 |
+
#
|
| 4265 |
+
# The derived overlay (`_derived_profile_columns`) decides by COMPARING the profile cell to the
|
| 4266 |
+
# fold of the series. The pass above REPAIRS THE SERIES β it retypes the child columns and runs
|
| 4267 |
+
# `_IG_RETYPE_CONVERTERS`, which rewrites `avg_engagement` from the vendor's 0-1 fraction to our
|
| 4268 |
+
# 0-100 percent. So on a tenant whose children are stale, the first proof compares a repaired
|
| 4269 |
+
# profile cell against an UNREPAIRED observation, finds them different, and declines to convert
|
| 4270 |
+
# a column that is in fact perfectly derivable. The next call then converts it β and
|
| 4271 |
+
# `verify_automation`'s "a SECOND pass is a no-op" check went red, which is exactly what that
|
| 4272 |
+
# check is for. MEASURED on the wave-26 fixture: 24 columns converted on pass 2, none on pass 1.
|
| 4273 |
+
#
|
| 4274 |
+
# β The module already knew this shape one level up β the profile loop's own note says the
|
| 4275 |
+
# schema pass "is safe only after those conversions, because the stored old type is the
|
| 4276 |
+
# idempotency key". Same argument, one table further down.
|
| 4277 |
+
# β CHEAP WHEN THERE IS NOTHING TO DO: on a current tenant the contract recomputes to the same
|
| 4278 |
+
# declarations and `_reconcile_ig_graph_fields` writes nothing. Twice, not looped: the repair is
|
| 4279 |
+
# what moves the data, and it has already run.
|
| 4280 |
+
wanted2, drops2 = _ig_schema_contract(rt)
|
| 4281 |
+
stats["schema"] = int(bool(stats["schema"]
|
| 4282 |
+
| int(_reconcile_ig_graph_fields(rt, wanted2, drops2))))
|
| 4283 |
# ββ 2026-08-10 β THE TWO VALUE BACKFILLS, and they run AFTER the schema pass on purpose:
|
| 4284 |
# `backfill_post_snapshot_grain` writes into columns that pass has just declared, and running
|
| 4285 |
# it first would find nothing to fill and report a truthful zero about the wrong moment.
|
platform/core/user_tables.py
CHANGED
|
@@ -1780,6 +1780,25 @@ def patch_profile_cell(table_key, row_id, fkey, value, st=None):
|
|
| 1780 |
|
| 1781 |
|
| 1782 |
def delete_row(table_key, row_id, st=None):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1783 |
if not is_user_table(table_key, st) or not records_mutable(table_key, st):
|
| 1784 |
return False
|
| 1785 |
|
|
@@ -1789,7 +1808,7 @@ def delete_row(table_key, row_id, st=None):
|
|
| 1789 |
(t.get('rows') or {}).pop(str(row_id), None)
|
| 1790 |
return cur
|
| 1791 |
|
| 1792 |
-
_st(st).update(STORE_KEY, _drop, flush='
|
| 1793 |
return True
|
| 1794 |
|
| 1795 |
|
|
|
|
| 1780 |
|
| 1781 |
|
| 1782 |
def delete_row(table_key, row_id, st=None):
|
| 1783 |
+
"""β SYNC, AND IT WAS `async` β owner report 2026-08-10: *"it took me forever to delete a
|
| 1784 |
+
record⦠it took me like 3 tries"*.
|
| 1785 |
+
|
| 1786 |
+
`flush='async'` applies the drop to the in-process CACHE and defers the upload by
|
| 1787 |
+
`store._FLUSH_DELAY` (2 s) so a burst coalesces into one commit. That trade was made for
|
| 1788 |
+
`add_row`, and made there for a stated reason β the owner's *"adding a new record visually
|
| 1789 |
+
takes too long, I need to be able to spam it"*. **This function inherited the flag and none of
|
| 1790 |
+
the argument.** Nobody spams a delete: it is rare, it is destructive, and it is the one
|
| 1791 |
+
mutation where "eventually consistent" is indistinguishable from "it did not work". Inside
|
| 1792 |
+
that window the row is gone from this process and still present in the store, so anything that
|
| 1793 |
+
re-reads the hub copy β a restart, an eviction, another reader β brings it back, and the user
|
| 1794 |
+
deletes it again.
|
| 1795 |
+
|
| 1796 |
+
β ONE COMMIT PER DELETE is the whole cost, and it is the correct one to pay: the alternative
|
| 1797 |
+
is a destructive action whose durability depends on the container staying up for two seconds.
|
| 1798 |
+
Adds keep their async path untouched β the spam case is real and is theirs.
|
| 1799 |
+
β THIS IS NOT THE WHOLE OF D-118. A sync commit closes the lost-delete window; it does not make
|
| 1800 |
+
two writers safe, which is D-4's job.
|
| 1801 |
+
"""
|
| 1802 |
if not is_user_table(table_key, st) or not records_mutable(table_key, st):
|
| 1803 |
return False
|
| 1804 |
|
|
|
|
| 1808 |
(t.get('rows') or {}).pop(str(row_id), None)
|
| 1809 |
return cur
|
| 1810 |
|
| 1811 |
+
_st(st).update(STORE_KEY, _drop, flush='sync')
|
| 1812 |
return True
|
| 1813 |
|
| 1814 |
|