fsanyoto commited on
Commit
1a96df4
Β·
verified Β·
1 Parent(s): 39368c0

Deploy AIOS web (React glide grid + FastAPI slice)

Browse files
RELEASES.json CHANGED
@@ -1,5 +1,5 @@
1
  {
2
- "current": "87826f5",
3
  "releases": [
4
  {
5
  "version": "v24",
 
1
  {
2
+ "current": "b03615b",
3
  "releases": [
4
  {
5
  "version": "v24",
VERSION CHANGED
@@ -1 +1 @@
1
- 87826f5
 
1
+ b03615b
api/automation_engine.py CHANGED
@@ -3429,7 +3429,25 @@ def _ig_schema_contract(rt, profile_tables=None):
3429
  continue
3430
  label = str(table.get("label") or table_key)
3431
  backlinks.append(_profile_backlink_field(table_key, label, bound))
3432
- wanted[table_key] = _profile_schema_for(bound)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3433
  # `tracked` was the pre-Stage progress checkbox. Keeping both allows two progress states
3434
  # to disagree, so it is retired by key just like the old nested Posts JSON.
3435
  drops[table_key] = {"posts", "post_hashtags", "tracked"}
@@ -3985,6 +4003,126 @@ def backfill_post_snapshot_grain(rt, log=print):
3985
  return len(changes)
3986
 
3987
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3988
  def migrate_ig_tables(rt, log=print, only=""):
3989
  """Bring every Instagram Profile and canonical child table onto one current contract.
3990
 
@@ -4121,6 +4259,27 @@ def migrate_ig_tables(rt, log=print, only=""):
4121
  # the stored old type is the idempotency key for unit-changing values such as engagement.
4122
  wanted, drops = _ig_schema_contract(rt)
4123
  stats["schema"] = int(_reconcile_ig_graph_fields(rt, wanted, drops))
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4124
  # ⭐⭐ 2026-08-10 β€” THE TWO VALUE BACKFILLS, and they run AFTER the schema pass on purpose:
4125
  # `backfill_post_snapshot_grain` writes into columns that pass has just declared, and running
4126
  # it first would find nothing to fill and report a truthful zero about the wrong moment.
 
3429
  continue
3430
  label = str(table.get("label") or table_key)
3431
  backlinks.append(_profile_backlink_field(table_key, label, bound))
3432
+ schema = _profile_schema_for(bound)
3433
+ # ⭐⭐ 2026-08-10 (owner: *"retire the old hardcoded method and replace the Instagram
3434
+ # database work with correct Rollup and Link fields"*) β€” THE DERIVED OVERLAY, AND IT
3435
+ # BELONGS HERE RATHER THAN IN A SEPARATE MIGRATION.
3436
+ #
3437
+ # β›” A standalone converter would LOSE. `_reconcile_ig_graph_fields` overwrites every
3438
+ # contract key it owns β€” `rollup` included, by its own note β€” so a column converted beside
3439
+ # the contract is re-typed back to `int` on the next reconcile, silently, hours later.
3440
+ # Making the CONTRACT itself say "this column is derived" leaves exactly one writer of the
3441
+ # schema instead of two that disagree.
3442
+ #
3443
+ # ⚠ THE SET IS PER TENANT AND IS A PROOF, NOT A LIST. `_derived_profile_columns` converts a
3444
+ # column only where this tenant's own rows show the fold already equals the stored value β€”
3445
+ # measured, because a named list would have blanked 221 live cells on nurilab alone
3446
+ # (`avg_engagement` and `category`, 62 each). A tenant whose series is thinner converts
3447
+ # fewer columns and loses nothing; as its history fills in, later passes convert more.
3448
+ derived = set(_derived_profile_columns(table, tables.get(IG_SNAPSHOTS_TABLE) or {}))
3449
+ wanted[table_key] = [_as_derived_field(f) if str(f.get("key")) in derived else f
3450
+ for f in schema]
3451
  # `tracked` was the pre-Stage progress checkbox. Keeping both allows two progress states
3452
  # to disagree, so it is retired by key just like the old nested Posts JSON.
3453
  drops[table_key] = {"posts", "post_hashtags", "tracked"}
 
4003
  return len(changes)
4004
 
4005
 
4006
+ #: ⭐⭐ 2026-08-10 (owner: *"retire the old hardcoded method and replace the Instagram database work
4007
+ #: with correct Rollup and Link fields"*) β€” THE PROFILE COLUMNS THAT MAY BECOME ROLLUPS.
4008
+ #:
4009
+ #: A column qualifies when the SERIES has somewhere to put it: `SNAPSHOT_FIELDS` carries the same
4010
+ #: key, so `latest` over `profile_snapshots_link` can re-derive it. That is 27 of the 44 preset
4011
+ #: columns β€” the other 17 are identity/bookkeeping (`handle`, `platform`, `enriched_at`), already
4012
+ #: relational (the 12 links and rollups), or have no observation behind them at all
4013
+ #: (`location_guess`/`location_confidence`, which WE infer rather than read).
4014
+ #:
4015
+ #: β›”β›” AND "DERIVABLE" IS NOT "SAFE", WHICH IS THE WHOLE REASON THIS IS A PREDICATE AND NOT A LIST.
4016
+ #: MEASURED on nurilab before any conversion: a bare `latest` would BLANK 221 filled cells β€”
4017
+ #: `avg_engagement` and `category` lose 62 each β€” because the discovery reads that produced those
4018
+ #: values were never recorded as observations (the hole this same wave closed going forward, in its
4019
+ #: historical form: 62 profiles hold exactly ONE observation, the enrichment scrape, dated three
4020
+ #: days AFTER the discovery run whose numbers are on their row). A migration that converts on a
4021
+ #: NAMED LIST would do that damage on any tenant whose series is thinner than the list's author
4022
+ #: assumed.
4023
+ #:
4024
+ #: β‡’ So the migration converts a column only where it can PROVE, on this tenant's own rows, that
4025
+ #: the derived value equals the stored one everywhere. See `_convertible_profile_columns`.
4026
+ #: ⚠ Re-runnable by design: as the series fills in, later passes convert more. A column that is not
4027
+ #: safe today is not refused forever, it is simply not converted yet.
4028
+ IG_DERIVABLE_KEYS = tuple(
4029
+ fd["key"] for fd in PRESET_PROFILE_FIELDS
4030
+ if fd.get("type") not in ("link", "rollup")
4031
+ and fd["key"] not in ("platform", "handle", "enriched_at", "first_found", "last_found",
4032
+ "found_count", "created_by")
4033
+ and fd["key"] in {s["key"] for s in SNAPSHOT_FIELDS})
4034
+
4035
+
4036
+ def _derived_profile_bag(key):
4037
+ """The rollup that replaces the mapper's write of ONE profile column.
4038
+
4039
+ ⚠ `where: is_not_empty` IS NOT DECORATION. A bare `latest` returns the newest observation's
4040
+ value INCLUDING ITS BLANK, so a cheap pull that did not read the field would erase what an
4041
+ expensive one learned β€” the exact asymmetry `upsert_rows`' merge exists to prevent on the
4042
+ scalar side. `where` runs BEFORE the ranking (contract C1), so this reads "the newest
4043
+ observation that ACTUALLY READ this field", which is what the materialised cell meant.
4044
+ MEASURED: it rescues 5 of the 12 columns a bare `latest` would damage; the other 7 need the
4045
+ observation itself, which is why the guard below is a proof and not a hope.
4046
+ """
4047
+ return {"link": "profile_snapshots_link", "field": key, "fn": "latest",
4048
+ "sortBy": "pulled_at", "sortDir": "desc",
4049
+ "where": [{"field": key, "op": "is_not_empty"}], "whereConj": "and"}
4050
+
4051
+
4052
+ def _derived_profile_columns(table, snapshots):
4053
+ """Which of `IG_DERIVABLE_KEYS` should be DECLARED as rollups on THIS table.
4054
+
4055
+ = the ones already converted, PLUS the ones whose derived value equals the stored value on
4056
+ every row. Returns keys.
4057
+
4058
+ β›”β›” IT MUST RE-ASSERT THE ONES ALREADY CONVERTED, and forgetting that is a one-line revert with
4059
+ a several-hour fuse. `_reconcile_ig_graph_fields` overwrites every contract key it owns β€”
4060
+ `rollup` included, by design β€” so a converted column that this function stopped naming would be
4061
+ re-typed back to `int`/`text` on the next reconcile, its bag dropped, and its value re-written
4062
+ by the mapper. The edit would look applied and undo itself later, which is exactly
4063
+ [[preset-unlock-needs-a-custody-stamp]]'s shape. An already-converted column is therefore
4064
+ included unconditionally rather than re-proved: its cells ARE the derived values, so a proof
4065
+ would be comparing the fold against itself.
4066
+
4067
+ β›” THE PROOF RUNS PER TENANT, PER COLUMN, AT MIGRATION TIME β€” never against a list somebody
4068
+ measured on one workspace. `followers` derives exactly on nurilab and could be thin somewhere
4069
+ else; `category` is damaged on nurilab and might be perfect elsewhere. The only honest
4070
+ authority is the data in front of the migration.
4071
+ ⚠ A column with NO stored values anywhere converts too, and that is deliberate rather than an
4072
+ oversight: there is nothing to lose, and leaving it materialised would mean the schema differs
4073
+ between two tenants for no reason a reader could discover.
4074
+ """
4075
+ fields = {str(f.get("key")): f for f in (table or {}).get("fields") or []}
4076
+ rows = (table or {}).get("rows") or {}
4077
+ by_subject = {}
4078
+ for snap in ((snapshots or {}).get("rows") or {}).values():
4079
+ key = str((snap or {}).get("influencer_key") or "").strip().lower()
4080
+ if key:
4081
+ by_subject.setdefault(key, []).append(snap or {})
4082
+ # The engine's own ordering, once per subject β€” newest first, unrankable rows last.
4083
+ ordered = {}
4084
+ for subject, observations in by_subject.items():
4085
+ keyed = [(o, _sort_key(o.get("pulled_at"), "date")) for o in observations]
4086
+ rankable = [(o, k) for o, k in keyed if k is not None]
4087
+ rankable.sort(key=lambda ok: ok[1], reverse=True)
4088
+ ordered[subject] = [o for o, _k in rankable] + [o for o, k in keyed if k is None]
4089
+ # β›” A rollup needs the LINK it folds. A profile table that has not been through the graph
4090
+ # reconcile has no `profile_snapshots_link`, and declaring one there would mint a column whose
4091
+ # link resolves to nothing β€” a blank cell wearing a configured column's clothes.
4092
+ if "profile_snapshots_link" not in fields:
4093
+ return []
4094
+ out = []
4095
+ for key in IG_DERIVABLE_KEYS:
4096
+ field = fields.get(key)
4097
+ if not field:
4098
+ continue
4099
+ if field.get("type") == "rollup":
4100
+ out.append(key) # already converted β€” see the note above on why
4101
+ continue
4102
+ declared = str(field.get("type") or "text")
4103
+ safe = True
4104
+ for row in rows.values():
4105
+ stored = str((row or {}).get(key) or "").strip()
4106
+ window = [o for o in ordered.get(
4107
+ str((row or {}).get("handle") or "").strip().lower(), [])
4108
+ if str(o.get(key) or "").strip() != ""] # the `where` guard, applied here
4109
+ derived = str(_rollup_fold("latest", [o.get(key) for o in window], declared)
4110
+ or "").strip()
4111
+ if stored != derived:
4112
+ safe = False
4113
+ break
4114
+ if safe:
4115
+ out.append(key)
4116
+ return out
4117
+
4118
+
4119
+ def _as_derived_field(field):
4120
+ """One preset scalar declaration β†’ the same column declared as a rollup over the series."""
4121
+ key = str(field.get("key") or "")
4122
+ return {**{k: v for k, v in field.items() if k not in ("agg",)},
4123
+ "type": "rollup", "rollup": _derived_profile_bag(key)}
4124
+
4125
+
4126
  def migrate_ig_tables(rt, log=print, only=""):
4127
  """Bring every Instagram Profile and canonical child table onto one current contract.
4128
 
 
4259
  # the stored old type is the idempotency key for unit-changing values such as engagement.
4260
  wanted, drops = _ig_schema_contract(rt)
4261
  stats["schema"] = int(_reconcile_ig_graph_fields(rt, wanted, drops))
4262
+ # β›”β›” A SECOND CONTRACT PASS, AND IT IS NOT BELT-AND-BRACES β€” WITHOUT IT THIS FUNCTION IS NOT
4263
+ # IDEMPOTENT, which is the one property everything else here is built on.
4264
+ #
4265
+ # The derived overlay (`_derived_profile_columns`) decides by COMPARING the profile cell to the
4266
+ # fold of the series. The pass above REPAIRS THE SERIES β€” it retypes the child columns and runs
4267
+ # `_IG_RETYPE_CONVERTERS`, which rewrites `avg_engagement` from the vendor's 0-1 fraction to our
4268
+ # 0-100 percent. So on a tenant whose children are stale, the first proof compares a repaired
4269
+ # profile cell against an UNREPAIRED observation, finds them different, and declines to convert
4270
+ # a column that is in fact perfectly derivable. The next call then converts it β€” and
4271
+ # `verify_automation`'s "a SECOND pass is a no-op" check went red, which is exactly what that
4272
+ # check is for. MEASURED on the wave-26 fixture: 24 columns converted on pass 2, none on pass 1.
4273
+ #
4274
+ # ⚠ The module already knew this shape one level up β€” the profile loop's own note says the
4275
+ # schema pass "is safe only after those conversions, because the stored old type is the
4276
+ # idempotency key". Same argument, one table further down.
4277
+ # ⚠ CHEAP WHEN THERE IS NOTHING TO DO: on a current tenant the contract recomputes to the same
4278
+ # declarations and `_reconcile_ig_graph_fields` writes nothing. Twice, not looped: the repair is
4279
+ # what moves the data, and it has already run.
4280
+ wanted2, drops2 = _ig_schema_contract(rt)
4281
+ stats["schema"] = int(bool(stats["schema"]
4282
+ | int(_reconcile_ig_graph_fields(rt, wanted2, drops2))))
4283
  # ⭐⭐ 2026-08-10 β€” THE TWO VALUE BACKFILLS, and they run AFTER the schema pass on purpose:
4284
  # `backfill_post_snapshot_grain` writes into columns that pass has just declared, and running
4285
  # it first would find nothing to fill and report a truthful zero about the wrong moment.
platform/core/user_tables.py CHANGED
@@ -1780,6 +1780,25 @@ def patch_profile_cell(table_key, row_id, fkey, value, st=None):
1780
 
1781
 
1782
  def delete_row(table_key, row_id, st=None):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1783
  if not is_user_table(table_key, st) or not records_mutable(table_key, st):
1784
  return False
1785
 
@@ -1789,7 +1808,7 @@ def delete_row(table_key, row_id, st=None):
1789
  (t.get('rows') or {}).pop(str(row_id), None)
1790
  return cur
1791
 
1792
- _st(st).update(STORE_KEY, _drop, flush='async')
1793
  return True
1794
 
1795
 
 
1780
 
1781
 
1782
  def delete_row(table_key, row_id, st=None):
1783
+ """β›” SYNC, AND IT WAS `async` β€” owner report 2026-08-10: *"it took me forever to delete a
1784
+ record… it took me like 3 tries"*.
1785
+
1786
+ `flush='async'` applies the drop to the in-process CACHE and defers the upload by
1787
+ `store._FLUSH_DELAY` (2 s) so a burst coalesces into one commit. That trade was made for
1788
+ `add_row`, and made there for a stated reason β€” the owner's *"adding a new record visually
1789
+ takes too long, I need to be able to spam it"*. **This function inherited the flag and none of
1790
+ the argument.** Nobody spams a delete: it is rare, it is destructive, and it is the one
1791
+ mutation where "eventually consistent" is indistinguishable from "it did not work". Inside
1792
+ that window the row is gone from this process and still present in the store, so anything that
1793
+ re-reads the hub copy β€” a restart, an eviction, another reader β€” brings it back, and the user
1794
+ deletes it again.
1795
+
1796
+ ⚠ ONE COMMIT PER DELETE is the whole cost, and it is the correct one to pay: the alternative
1797
+ is a destructive action whose durability depends on the container staying up for two seconds.
1798
+ Adds keep their async path untouched β€” the spam case is real and is theirs.
1799
+ ⚠ THIS IS NOT THE WHOLE OF D-118. A sync commit closes the lost-delete window; it does not make
1800
+ two writers safe, which is D-4's job.
1801
+ """
1802
  if not is_user_table(table_key, st) or not records_mutable(table_key, st):
1803
  return False
1804
 
 
1808
  (t.get('rows') or {}).pop(str(row_id), None)
1809
  return cur
1810
 
1811
+ _st(st).update(STORE_KEY, _drop, flush='sync')
1812
  return True
1813
 
1814