diff --git "a/api/automation_engine.py" "b/api/automation_engine.py" --- "a/api/automation_engine.py" +++ "b/api/automation_engine.py" @@ -158,6 +158,45 @@ APPEND_TABLES = frozenset({IG_SNAPSHOTS_TABLE, IG_POST_SNAPSHOTS_TABLE, #: lands here IN THE SAME CHANGE as its `RUNNERS` entry and its flip law: a kind in this tuple #: with no runner is a control that must refuse. KINDS = ("plain", "scrape_db", "field_instagram", "discover_instagram", "discover_tiktok") +#: ⭐⭐ WAVE 30 · T04 — THE DISCOVERY KINDS UNDER ONE NAME, and this constant is a bug fix rather +#: than tidying. `discover_tiktok` shipped in wave 29 by being added to `KINDS`, `RUNNERS`, +#: `clean_config` and `TRIGGER_*` — four sites that were found — while FIVE more tested the string +#: `"discover_instagram"` directly and were not. The visible symptom was the owner's: picking the +#: TikTok trigger 400'd with *"say how many profiles to fetch"* on the FIRST save, because the seed +#: below was one of the five. The other four are silent — a canvas with no `find` panel, a flow +#: table resolving to the wrong default, and a summary sentence announcing the automation would +#: "do nothing yet". +#: +#: ⛔ SO THE RULE IS: A DISCOVERY BRANCH TESTS MEMBERSHIP OF THIS TUPLE, NEVER A KIND STRING. +#: Three parallel string comparisons is precisely how the third platform gets missed twice more, +#: and the misses are individually invisible — each one degrades a different surface, none of them +#: raises, and every gate stays green (this whole family shipped green in wave 29). +#: ⚠ The kind ↔ platform facts that genuinely DIFFER — the dataset id, the default target table, +#: the field map — stay resolved per kind where they are used. This tuple answers *"is this a +#: corpus search?"* and nothing else; widening it into a platform registry would just move the +#: problem somewhere with a longer name. +DISCOVERY_KINDS = ("discover_instagram", "discover_tiktok") +#: ⭐ WAVE 30 · T06 — THE TRIGGER HALF, and it is a SEPARATE map because the two are NOT +#: interchangeable, which cost a red to learn. Law 1 makes a discovery TRIGGER choose its kind, so +#: `trigger ⇒ kind` always holds — but the converse does NOT: a definition can carry +#: `kind: "discover_instagram"` with no trigger at all (a direct API create does exactly that, and +#: `verify_automation`'s `_discover_defn` fixture is one). Testing the KIND where the rule is about +#: the trigger therefore fires on definitions the picker could never produce — measured: it spawned +#: a preset database under a dry-run fixture that asserts none exists. +#: ⛔ SO: "did somebody PICK a corpus search in the picker?" reads THIS. "Is this stored definition a +#: corpus search?" reads `DISCOVERY_KINDS`. Two questions, two maps, and the gate asserts this one +#: agrees with law 1 rather than trusting that it does. +DISCOVERY_TRIGGER_KIND = {"ig_profile_match": "discover_instagram", + "tiktok_profile_match": "discover_tiktok"} +#: ⭐ WAVE 30 · T08 — the ENRICH action kinds, one per network. Same argument as `DISCOVERY_KINDS` +#: one paragraph up: these two share a validator, a selection, a cooldown and a run summary, and the +#: only things that differ are which connector answers and which tables the rows land in. +#: ⛔ ONE VALIDATOR, NOT TWO. `clean_actions`' enrich branch is ~40 lines of clamps whose comments +#: record why each one is shaped the way it is (`submitted=False` because the client re-posts the +#: whole action list; `limit` clamped twice because the run sees STORED configs). A second copy for +#: TikTok would start identical and drift, and the way it fails is that one network silently accepts +#: a limit the other refuses. +ENRICH_KINDS = ("enrich_instagram", "enrich_tiktok") #: What a definition with no kind becomes (C-TRIG law 2) — the shape `POST /automations` stores #: when the body names none, which after R6 is every create the client makes. DEFAULT_KIND = "plain" @@ -1535,8 +1574,12 @@ def clean_config(kind, raw, previous=None): # kind rather than hard-coded — a TikTok search falling back to `ut_ig_profile` would write # TikTok rows into the Instagram family, which is precisely what R2's parallel family exists # to prevent, and it would do it silently. - if kind in ("discover_instagram", "discover_tiktok"): - tiktok = kind == "discover_tiktok" + if kind in DISCOVERY_KINDS: + # ⭐ WAVE 30 · T05 — the inline tuple became the named one, and the two defaults below now + # come from `discovery_facts` rather than being spelled out here. They were correct; they + # were also the FIFTH copy of "which table does this kind write to", and the other four + # were the ones wave 29 forgot to update. + _, _disc_table, _disc_label, _ = discovery_facts(kind) limit = int(raw.get("recordsLimit") or 0) if limit < 1: return None, ("say how many profiles to fetch — an UNBOUNDED discovery query is the " @@ -1551,8 +1594,7 @@ def clean_config(kind, raw, previous=None): preds, perr = clean_predicates(raw.get("predicates"), join) if perr: return None, perr - target = (_s(raw.get("targetTable"), 60).strip() - or (TT_PROFILE_TABLE if tiktok else DISCOVER_TABLE)) + target = _s(raw.get("targetTable"), 60).strip() or _disc_table if not target.startswith(UT_PREFIX): target = UT_PREFIX + target seed, serr = clean_seed(raw.get("seed") if "seed" in raw else prev.get("seed")) @@ -1561,9 +1603,7 @@ def clean_config(kind, raw, previous=None): return {"recordsLimit": limit, "predicates": preds, "operator": join, "targetTable": target, - "targetLabel": (_s(raw.get("targetLabel"), 60).strip() - or (TT_TABLE_LABELS[TT_PROFILE_TABLE] if tiktok - else DISCOVER_LABEL)), + "targetLabel": _s(raw.get("targetLabel"), 60).strip() or _disc_label, # C6/R5: WHERE the conditions above came from, when they were derived. Stored so # the surface can say "these were filled in from the 'Florists' view, over 12 # records" instead of presenting them as if somebody typed them. @@ -1979,7 +2019,21 @@ def clean_definition(raw, previous=None, username=""): if not name: return None, "name the automation" cfg_raw = raw.get("config") if isinstance(raw.get("config"), dict) else prev.get("config") - if kind == "discover_instagram" and prev.get("kind") != "discover_instagram": + if kind in DISCOVERY_KINDS and prev.get("kind") != kind: + # ⭐⭐ WAVE 30 · T04 — WIDENED FROM `discover_instagram` TO BOTH DISCOVERY KINDS, AND THIS + # ONE LINE WAS THE WHOLE OF THE OWNER'S ITEM 3. Picking "When a TikTok profile fits a + # criteria" 400'd on the very first Save with *"say how many profiles to fetch — an + # UNBOUNDED discovery query is the one shape the vendor refuses outright + # (NOT_ENOUGH_FUNDS)"*, and the trigger was therefore never STORED, which is what the + # owner saw as "the Trigger won't load". + # ⛔ INSTAGRAM WAS NEVER SURVIVING ON ITS OWN MERITS: `AutomationDetail.buildConfig` sends + # no `recordsLimit` for EITHER platform on a fresh automation (its `kind` is still `plain` + # at that moment, so it falls through to `return { targetTable }`). IG worked only because + # this seed caught it. So the defect was never "TikTok is missing something Instagram has" + # — it was one hard-coded string in the single line that rescues both. + # ⚠ `prev.get("kind") != kind` is the EXACT generalisation of the old + # `prev.get("kind") != "discover_instagram"`, not a loosening: it still fires only on a + # kind FLIP, so a later save that carries a real limit is not re-seeded (asserted). # AMENDMENT A1: the kind FLIP seeds the one field `clean_config` insists on, or the very # first Save after picking this trigger 400s — against the stored-inert-with- # `configured:false` pattern the whole picker is built on (see `clean_trigger`'s A3 note). @@ -2420,8 +2474,25 @@ def _presets_after_write(rt, defn, username): if not target: return try: - if (defn.get("trigger") or {}).get("key") == "ig_profile_match": - ut_ensure(rt, cfg.get("targetLabel") or DISCOVER_LABEL, CANDIDATE_FIELDS, username, + # ⭐⭐ WAVE 30 · T06 — BOTH DISCOVERY KINDS SPAWN ON SAVE, and TikTok's absence here was + # R10 simply not holding for the second platform: `TT_PROFILE_FIELDS` reached `ut_ensure` + # at exactly ONE site — inside `run_discover_tiktok` — so the database did not exist until + # money had already been spent finding out what was in it. That is the precise pre-R10 + # behaviour `verify_automation`'s NC39 forbids on the Instagram side. + # ⛔⛔ THE DISCRIMINATOR IS THE **TRIGGER**, NOT THE KIND, AND THE GATE PAID FOR THAT + # SENTENCE. Widening this to `kind in DISCOVERY_KINDS` looks equivalent — law 1 makes a + # discovery trigger choose its kind — but the implication only runs ONE WAY. A definition + # may carry `kind: "discover_instagram"` with **no trigger at all**: law 1 sets the kind + # from the trigger and never the reverse, so any direct API create can do it, and one of + # this repo's own fixtures does. MEASURED: the kind test spawned a preset database under a + # dry-run fixture whose check asserts that no table exists — a red on correct-looking code, + # caught only because that check happened to exist. + # ⇒ R10 is a rule about what somebody PICKED IN THE PICKER, so it reads the picker's own + # answer. `DISCOVERY_TRIGGER_KIND` is that map, and a gate asserts it agrees with law 1. + _dkind = DISCOVERY_TRIGGER_KIND.get((defn.get("trigger") or {}).get("key") or "") + if _dkind: + _, _, _spawn_label, _spawn_fields = discovery_facts(_dkind) + ut_ensure(rt, cfg.get("targetLabel") or _spawn_label, _spawn_fields, username, key=target, flow_tag=str(defn.get("id") or ""), lock_fields=True) return enrich_actions = _actions_of_kind(((defn.get("flow") or {}).get("actions") or []), @@ -2435,6 +2506,38 @@ def _presets_after_write(rt, defn, username): if enrich_actions and bound: ut_ensure(rt, table.get("label") or target, _profile_schema_for(bound), username, key=target, flow_tag=str(defn.get("id") or ""), lock_fields=True) + # ⭐ WAVE 30 · T08 — the TikTok half of W25/R10: the columns an `enrich_tiktok` step will + # fill appear when the automation is SAVED, not when money is first spent. Resolved + # through `profile_field_key(..., source=PROFILE_SOURCE_TT)` so it cannot adopt an + # Instagram binding, and through the SAME `_tt_profile_schema_for` the runner's own top-up + # uses — R10's rule is that the column set does not change between Save and the first run, + # and one shared resolver is what makes that structural instead of a convention. + tt_actions = _actions_of_kind(((defn.get("flow") or {}).get("actions") or []), + "enrich_tiktok") + if tt_actions: + tt_named = next((str((a.get("config") or {}).get("profileField") or "").strip() + for a in tt_actions + if str((a.get("config") or {}).get("profileField") or "").strip()), "") + tt_bound = profile_field_key(table, tt_named, source=PROFILE_SOURCE_TT) + if tt_bound: + ut_ensure(rt, table.get("label") or target, _tt_profile_schema_for(tt_bound), + username, key=target, flow_tag=str(defn.get("id") or ""), + lock_fields=True) + # ⭐ WAVE 30 · T10 — and the CHILD databases the step's own switches promise, at + # SAVE, before any money. R10's rule is "the columns a step will fill appear when + # you press Save"; a `postMetrics` toggle whose database only exists after a paid + # run is the same complaint one level up. + # ⛔ GATED ON THE SWITCHES, never spawned unconditionally: a database that exists + # because somebody looked at a toggle, and then never fills, is the "SECOND, empty + # database" the discovery default was rewritten to stop producing. + tt_cfgs = [(a.get("config") or {}) for a in tt_actions] + for _flag, _child in (("postMetrics", TT_POSTS_TABLE), + ("postMetrics", TT_POST_SNAPSHOTS_TABLE), + ("commentMetrics", TT_COMMENTS_TABLE)): + if any(bool(c.get(_flag)) for c in tt_cfgs): + ut_ensure(rt, TT_TABLE_LABELS[_child], TT_TABLE_FIELDS[_child], username, + key=_child, flow_tag=str(defn.get("id") or ""), + lock_fields=True) except Exception as e: # noqa: BLE001 # ⛔ NEVER FAILS THE SAVE. The definition is already committed by the time this runs, so # raising here would answer 500 for an automation that IS stored — the caller would retry @@ -2893,11 +2996,12 @@ POST_SNAPSHOT_FIELDS = [ #: twin of `PROFILE_SOURCE_IG`, which is declared further down beside the `PLATFORM_*` vocabulary #: with the note on why those two families of string are NOT the same thing. It sits here rather #: than there because the lists below are evaluated at IMPORT and would raise a NameError otherwise. -#: ⛔ THIS CONSTANT IS ONLY HALF THE CONTRACT. The other half is one line in -#: `platform/core/user_tables.py` — `PROFILE_SOURCES` must gain `'tiktok'` — which lives in another -#: session's fence and is asked for in B's mailbox. Until that line lands, `_clean_profile` REFUSES -#: this flag and the handle column of a `ut_tt_profile` spawn is dropped. Fail-closed and loud, by -#: design: the alternative is a profile database whose profile column silently is not one. +#: ⛔ THIS CONSTANT IS ONLY HALF THE CONTRACT, AND THE OTHER HALF HAS LANDED (verified 2026-08-12, +#: W30-T08): `platform/core/user_tables.py:PROFILE_SOURCES` now reads `('instagram', 'tiktok')`. +#: Before it did, `_clean_profile` REFUSED this flag and the handle column of a `ut_tt_profile` +#: spawn was dropped — fail-closed and loud by design, because the alternative is a profile +#: database whose profile column silently is not one. Left written down rather than deleted: the +#: refusal is what a TikTok binding looks like on any deployment where that line is missing. PROFILE_SOURCE_TT = "tiktok" #: The profile row: LATEST values plus `enriched_at`. The SERIES lives in `ut_tt_snapshots` — @@ -3118,16 +3222,27 @@ TT_TABLE_LABELS = { # inside its functions, precisely so the two modules cannot cycle. This import sits BELOW the rail # and below `PACE_SECONDS`, so even if somebody later makes one of those reach-backs a # module-level import, the names it wants already exist and the cycle still resolves. +# ⭐⭐ WAVE 30 · T09 (D-128) — TWO IMPORT BLOCKS NOW, AND THE SPLIT BETWEEN THEM IS THE POINT. +# `connectors_bd` is the SUPPLIER's wire — it serves both platforms and knows neither. `connectors_ig` +# is INSTAGRAM's vocabulary: its dataset ids, its row mappers, its free rungs. Anything that would +# have to change to serve a third platform belongs in the second block, not the first. +# ⚠ `bd_ready` is re-exported through this module ON PURPOSE — `routes_automation` and +# `routes_connectors` both reach `engine.bd_ready()` on the module object, and those files belong to +# other sessions. Dropping it here would 500 two surfaces that never mention a vendor. +from connectors_bd import ( # noqa: E402 + BD_EXCLUDE_MAX, BD_PATH_SNAPSHOT, BD_RECORD_PRICE_SPEC, + _bd_deferral, _bd_first_url, _bd_rows, + _first, _ig_int, + bd_call, bd_filter_rows, bd_filter_start, bd_filter_status, bd_ready, + bd_snapshot_progress, depth_refusal, +) from connectors_ig import ( # noqa: E402 - BD_DS_COMMENTS, BD_DS_POSTS, BD_DS_PROFILES, BD_DS_REELS, BD_EXCLUDE_MAX, BD_PATH_SNAPSHOT, - BD_RECORD_PRICE_SPEC, - _bd_comment, _bd_deferral, _bd_first_url, _bd_post_metrics, _bd_profile, _bd_rows, + BD_DS_COMMENTS, BD_DS_POSTS, BD_DS_PROFILES, BD_DS_REELS, + _bd_comment, _bd_post_metrics, _bd_profile, _bd_tagged_location, - _first, _ig_int, DEFERRED_MARK, apify_profile, - bd_call, bd_filter_rows, bd_filter_start, bd_filter_status, bd_profiles_batch, bd_ready, - bd_snapshot_progress, depth_refusal, + bd_profiles_batch, ig_handle, public_source, pull_profile, select_post_groups, top_up_views, ) @@ -3531,6 +3646,46 @@ PLATFORM_TIKTOK = "TikTok" PLATFORM_FACEBOOK = "Facebook" PLATFORMS = (PLATFORM_INSTAGRAM, PLATFORM_TIKTOK, PLATFORM_FACEBOOK) + +def discovery_facts(kind): + """⭐⭐ WAVE 30 · T05 — THE THREE THINGS THAT GENUINELY DIFFER BETWEEN THE DISCOVERY KINDS: + the network's NAME, the database a run writes to when the config names none, and that + database's label. `(platform, table, label)`. + + ⛔ WHY A FUNCTION AND NOT A DICT LITERAL: `DISCOVER_TABLE` and `DISCOVER_LABEL` are declared + several hundred lines BELOW this point, so a dict evaluated here would NameError at import. + A function body resolves at call time and does not care. (`PROFILE_SOURCE_IG`'s note directly + above records the same import-order trap costing a constant its natural home.) + + ⛔ AND WHY IT EXISTS AT ALL. These three facts were written out inline at FIVE call sites — + `clean_config`, `graph`'s discovery arm, `graph`'s `det` dict, `_flow_table` and + `compose_sentence` — and wave 29 taught four of the five about TikTok by not touching them: + they tested the string `"discover_instagram"`, so a stored TikTok automation fell through to + a default meant for Instagram, or to no arm at all. Every one of those misses was SILENT and + every gate stayed green. `DISCOVERY_KINDS` answers *"is this a corpus search?"*; this answers + *"whose?"* — and between them a sixth platform is one tuple entry and one arm here, rather + than a hunt through the file for string comparisons somebody has to think to look for. + + ⚠ DELIBERATELY NOT A PLATFORM REGISTRY. The dataset ids, the field maps and the row mappers + stay in the connector modules that own them. This is the presentation-and-default quartet the + ENGINE needs, and widening it is how it becomes a second `SOURCES` with a longer name. + + ⭐ T06 ADDED THE FOURTH ELEMENT, `spawn_fields` — the field list `_presets_after_write` gives + the target database on SAVE. It belongs here rather than beside that function because it must + equal what the RUN-TIME `ut_ensure` uses, and wave 25's R10 exists precisely to stop those two + disagreeing: columns that change between Save and the first run are two different answers to + "what does this database look like". + ⚠ THE TWO PLATFORMS PASS DIFFERENT-LOOKING LISTS AND THAT IS CORRECT, not an oversight. + Instagram spawns `CANDIDATE_FIELDS` (= the preset set PLUS the discovery bookkeeping) because + its preset list alone is a subset of what its runner writes; TikTok's `TT_PROFILE_FIELDS` + already carries its own bookkeeping columns, so it IS the whole set. Each side is the list its + own runner ensures — which is the rule, rather than "both use the one named CANDIDATE". + """ + if kind == "discover_tiktok": + return (PLATFORM_TIKTOK, TT_PROFILE_TABLE, TT_TABLE_LABELS[TT_PROFILE_TABLE], + TT_PROFILE_FIELDS) + return PLATFORM_INSTAGRAM, DISCOVER_TABLE, DISCOVER_LABEL, CANDIDATE_FIELDS + #: ⭐ WAVE 26 �� R3 — THE TYPES ARE HONEST NOW, AND THE MIGRATION IS THE PRICE OF THAT. #: #: This block used to say the `text` types were deliberate, and the reasoning was sound as far as @@ -3801,6 +3956,31 @@ def _profile_schema_for(bound_key): return out +def _tt_profile_schema_for(bound_key): + """⭐ WAVE 30 · T08 — TikTok preset columns for a database whose profile column is `bound_key`. + + ⛔ NOT `_profile_schema_for` WITH A LIST ARGUMENT, and the difference is not stylistic. That + function walks `PRESET_PROFILE_FIELDS` and only ever removes the flag from `PRESET_FLAG_KEY`, + because on the Instagram side the flag lives on exactly one known column. TikTok's binding may + be ANY column a person named (`profile_field_key` step 1), so the rule here has to be stated + the other way round: **every field that is not the bound one arrives as DATA**, stripped of + both the identity flag and `pinned`. + ⚠ Otherwise a database whose TikTok column is `creator` would gain a rival `handle` carrying + `profile: {source: "tiktok"}` — a SECOND profile column, which `user_tables` refuses at both + write doors, so the whole top-up would be rejected and every preset cell would then be dropped + for want of a column. One shared helper for the save-time and run-time paths, so those two + cannot answer "which columns does a TikTok enrich need" differently. + """ + out = [] + for field in TT_PROFILE_FIELDS: + item = dict(field) + if str(item.get("key") or "") != str(bound_key or ""): + item.pop("profile", None) + item.pop("pinned", None) + out.append(item) + return out + + def _locked_ig_field(field): """One canonical Instagram field with the immutable preset declaration attached.""" item = dict(field) @@ -4086,17 +4266,67 @@ _MIGRATE_CONVERT = { } +#: ⭐⭐ WAVE 30 · T08 — THE COLUMNS THAT CAN ONLY BE TIKTOK'S, DERIVED rather than typed out, so +#: that adding a column to either declaration keeps this set correct instead of quietly emptying +#: it. Eight today: `tt_id`, `like_engagement`, `comment_engagement`, `likes_received`, `region`, +#: `predicted_lang`, `account_created_at`, `source`. +TT_ONLY_PROFILE_KEYS = (frozenset(f["key"] for f in TT_PROFILE_FIELDS) + - frozenset(PRESET_PROFILE_KEYS) + - frozenset(f["key"] for f in CANDIDATE_FIELDS)) + + +def _is_tt_profile_table(tbl): + """⭐⭐ WAVE 30 · T08 — is this database a TIKTOK profile table? + + ⛔ THE DEFECT THIS EXISTS TO CLOSE WAS LIVE AND SHIPPED, and it converted a customer's TikTok + database into an Instagram one on the SECOND write to it. MEASURED: a `ut_tt_profile` spawned + by W30-T06 comes out correct — `handle` carrying `profile: {source: "tiktok"}` — and one more + `ut_ensure` on that table (a re-save, the discovery runner's own ensure, the next run) leaves + it carrying `profile: {source: "instagram"}` and the description *"Instagram username without + the @ symbol"*. `profile_field_key(..., source="tiktok")` then answers `""` forever, so every + TikTok enrich step on that database reports UNBOUND, and every row gets stamped + `platform: Instagram` — which is half of W26/R4's `(platform, handle)` identity, so two + different people's accounts merge under one key with nothing red anywhere. + ⚠ It hid because the migration SKIPS a table that does not exist yet, so the spawn itself is + always clean and only the second write is not. A gate that creates and asserts once cannot see + it; the assertion has to survive a second `ut_ensure`. + + ⭐ TWO LEGS, and the second is the one that still works after the first has been eaten: + 1. the table SAYS SO — a column declaring `profile: {source: "tiktok"}` is the database's own + statement of which network it is about, and it is the same declaration `profile_field_key` + reads, so there is one answer to "whose profile table is this"; + 2. it declares a column only TikTok has. Needed because leg 1 is exactly what the defect + destroys: on a table already corrupted in production, the flag now says Instagram, and a + detector resting on it alone would agree with the corruption and keep re-applying it. + """ + fields = (tbl or {}).get("fields") or [] + for f in fields: + p = f.get("profile") + if isinstance(p, dict) and str(p.get("source") or "") == PROFILE_SOURCE_TT: + return True + return bool({f.get("key") for f in fields} & TT_ONLY_PROFILE_KEYS) + + def _ig_profile_tables(rt, tables=None): """Every `ut_*` table in this tenant that looks like an Instagram profile table. ``tables`` is the optional lent snapshot (W29-T01): read-only walk, so a caller that already holds the bucket must not pay for a second deep copy of it. + + ⛔ WAVE 30 · T08 — AND THE TEST IS NOW EXCLUSIVE, because "looks like" was network-blind and a + TikTok profile table looks EXACTLY like one: 18 of `ut_tt_profile`'s 30 columns are Instagram + preset keys, and the bar here is `handle` plus three. Every caller of this function then treats + the match as an Instagram table — `migrate_ig_tables` rewrites its declarations to Instagram's + contract, and `discover_default_table` offers it as the default target for an Instagram search. """ out = [] for key, tbl in sorted(((ut_all(rt) if tables is None else tables) or {}).items()): keys = {f.get("key") for f in (tbl or {}).get("fields") or []} - if "handle" in keys and len(keys & set(PRESET_PROFILE_KEYS)) >= MIGRATE_MIN_PRESET_FIELDS: - out.append(key) + if "handle" not in keys or len(keys & set(PRESET_PROFILE_KEYS)) < MIGRATE_MIN_PRESET_FIELDS: + continue + if _is_tt_profile_table(tbl): + continue + out.append(key) return out @@ -5672,6 +5902,142 @@ def preset_cells(res, pulled): return cells +#: Keys the TikTok SNAPSHOT row owns itself, so the copy loop below never overwrites them from the +#: profile map. Mirrors `_SNAPSHOT_OWN_KEYS` on the Instagram side. +_TT_SNAPSHOT_OWN_KEYS = ("snapshot_key", "influencer_key", "pulled_at", "source", "approx") + + +def tt_preset_cells(res, pulled): + """⭐⭐ WAVE 30 · T08 — one TikTok pull → the LATEST-value cells written onto the record. + + ⛔ THIS IS NOT `preset_cells` WITH A FLAG, AND THAT IS THE WHOLE DESIGN DECISION. Instagram's + `preset_cells` walks `PRESET_FROM_PROFILE` (an Instagram map), stamps + `cells["platform"] = PLATFORM_INSTAGRAM` UNCONDITIONALLY, and computes a location guess from + posts. Passing a TikTok row through it would stamp every TikTok creator as Instagram — which is + the identity half of W26's `(platform, handle)` key, so it would not merely mislabel a row, it + would MERGE two different people's accounts under one key. + ⭐ AND THE TIKTOK ROW DOES NOT NEED A MAP AT ALL. `connectors_tt.normalize_profile` already + emits our column names — it is the SAME function the discovery runner uses — so a scraped row + and a corpus row cannot disagree about which vendor key became which column. All this adds is + what the FETCH knows and the map cannot: when it was read, and by which route. + + ⚠ FILTERED TO THE DECLARED SCHEMA. `ut_write_rows`/`_clean_field` DROP an undeclared key with + no error and a successful-looking run (the defect `section_w29_tiktok_schema`'s emit-vs-declare + sweep exists for), so an unknown key is refused here where it is visible rather than swallowed + three layers down. + """ + prof = (res or {}).get("profile") or {} + declared = {f["key"] for f in TT_PROFILE_FIELDS} + cells = {k: v for k, v in prof.items() if k in declared} + # `source` is the ROUTE that answered, not the vendor's name — the same thing `via` carries on + # the Instagram snapshot, and the reason a stored measurement can always say how it was read. + if (res or {}).get("via"): + cells["source"] = _s((res or {}).get("via"), 60) + # R3, as on the Instagram side: a `date`, because "how stale is this number" is a question in + # days. Without it a blank `followers` means both "never enriched" and "enriched and empty". + cells["enriched_at"] = _day(pulled) + return cells + + +#: The metric columns a TikTok post SNAPSHOT carries. ⛔ DERIVED from the declaration, minus the +#: keys the snapshot owns itself — so adding a metric to `TT_POST_SNAPSHOT_FIELDS` starts being +#: captured, and adding an IDENTITY column to it never does. +_TT_PSNAP_OWN_KEYS = ("platform", "post_snapshot_key", "shortcode", "influencer_key", "pulled_at", + "post_link") +TT_PSNAP_METRIC_KEYS = tuple(f["key"] for f in TT_POST_SNAPSHOT_FIELDS + if f["key"] not in _TT_PSNAP_OWN_KEYS) + + +def tt_capture_rows(res, pulled): + """⭐⭐ WAVE 30 · T10 — one TikTok pull → `(post_identity_rows, post_metric_rows, comment_rows)`. + + The TikTok twin of `capture_rows`, and separate for the same reason `tt_preset_cells` is: that + function walks Instagram's `SNAPSHOT_FIELDS`/`PRESET_FROM_PROFILE` and stamps Instagram's + platform. The mappers have already done the vendor→our-keys work here + (`connectors_tt.normalize_post` / `normalize_comment`), so this adds only what the FETCH knows. + + ⚠ THE TWO WRITE DISCIPLINES ARE OPPOSITE, exactly as on the Instagram side: + * AN IDENTITY ROW OMITS ITS BLANKS, because it is UPSERTED — `upsert_rows` writes the keys it + is handed, so sending "" would let a thin run ERASE what a full one learned. + * A METRIC SNAPSHOT IS APPENDED ONLY WHEN SOMETHING WAS MEASURED. A row in a time series + should mean "this was true then"; a row meaning "nobody looked" eats the cap and draws a + chart of nothing. + + ⛔⛔ D-117 IS NOT REPRODUCED HERE, AND THAT IS THE INSTRUCTION THE TICKET LEADS WITH. + Instagram denormalises `posted_at` and `type` onto its post SNAPSHOT rows, where they never + reconcile against the identity table and quietly become a second, ageing answer to a question + the posts table already answers. This snapshot carries the METRICS and the join key and nothing + else — `TT_PSNAP_METRIC_KEYS` is derived from the declaration minus the keys the snapshot owns, + so the exclusion is structural rather than a list somebody has to remember to keep short. + """ + prof = (res or {}).get("profile") or {} + who = str(prof.get("handle") or "").strip().lstrip("@").lower() + idents, metric_rows, comment_rows = [], [], [] + for p in (res or {}).get("posts") or []: + shortcode = str((p or {}).get("shortcode") or "").strip() + if not shortcode: + continue + ident = {k: v for k, v in p.items() if v not in (None, "")} + # ⚠ The Posts dataset carries `profile_username`, but a row that omits it must still join: + # the profile whose `top_videos` we scraped IS the influencer, and that is a fact of the + # call rather than of the row. + ident.setdefault("influencer_key", who) + ident["platform"] = PLATFORM_TIKTOK + idents.append(ident) + measured = {k: p.get(k) for k in TT_PSNAP_METRIC_KEYS + if k != "source_payload" and p.get(k) is not None} + if not measured: + continue + row = {"platform": PLATFORM_TIKTOK, "shortcode": shortcode, + "influencer_key": str(p.get("influencer_key") or who), + "post_snapshot_key": f"{shortcode}@{pulled}", "pulled_at": pulled} + for k, v in measured.items(): + row[k] = _s(v, 400) + metric_rows.append(row) + for c in (res or {}).get("comments") or []: + if not str((c or {}).get("comment_key") or "").strip(): + continue + row = {k: v for k, v in c.items() if v not in (None, "")} + # Same fact-of-the-call argument: the Comments dataset has no influencer field at all, so + # without this the comments table could never be filtered by creator. + row.setdefault("influencer_key", who) + row["platform"] = PLATFORM_TIKTOK + comment_rows.append(row) + return idents, metric_rows, comment_rows + + +def tt_snapshot_row(res, pulled): + """One TikTok pull → its `ut_tt_snapshots` observation, or None when there is no handle. + + ⛔ THE APPEND LAW, UNCHANGED FROM INSTAGRAM AND FOR THE SAME MEASURED REASON: TikTok's Profiles + dataset carries **no measurement timestamp** (`create_time` is when the ACCOUNT was made, not + when `followers` was true — probed, `tiktok-capture.md`). So the series is dated by when WE + read it, and the key is `@` — idempotent by its KEY rather than by a flag, + so re-running over the same rows rewrites the same row instead of growing the table. + ⚠ `pulled_at` keeps the FULL stamp while the record's `enriched_at` is a day: the record answers + "how stale", the series answers "when exactly", and collapsing the second into the first would + make two reads on one day indistinguishable. + """ + prof = (res or {}).get("profile") or {} + handle = str(prof.get("handle") or "").strip().lstrip("@").lower() + if not handle: + return None + out = {"platform": PLATFORM_TIKTOK, "snapshot_key": f"{handle}@{pulled}", + "influencer_key": handle, "pulled_at": pulled, + "source": _s((res or {}).get("via") or "brightdata", 60)} + for fd in TT_SNAPSHOT_FIELDS: + key = fd["key"] + if key in _TT_SNAPSHOT_OWN_KEYS or key == "platform": + continue + value = prof.get(key) + # ⚠ A ZERO IS A MEASUREMENT AND SURVIVES — the test is `is not None`, never truthiness. + # `str(value or "").strip()` would drop a genuine 0 follower count, and this is a series + # whose whole purpose is that a number moved. + if value is not None and str(value).strip() != "": + out[key] = str(value) if key == "source_payload" else _s(value, 400) + return out + + def run_field_instagram(rt, defn, username="automation", log=print, step=_no_step, rows=None): """Automation #2 (R7): for every row of a database that carries a profile URL, pull the public @@ -5932,14 +6298,151 @@ def _auto_url_field(table, fkey): return "" -def run_discover_instagram(rt, defn, username="automation", log=print, step=_no_step, - rows=None): - """Automation #3 (owner ruling R7 / D-23): FIND handles nobody here has typed in. +#: ⭐ W29-T04 — the discovery-run bookkeeping columns, so `run_discover_tiktok` and the D-116 merge +#: rule agree on what discovery OWNS versus what enrichment owns. Anything not in this set is a +#: MEASUREMENT, and a corpus re-find must never overwrite an enriched measurement with its own +#: cheaper approximation. +TT_DISCOVERY_KEYS = frozenset({"platform", "handle", "created_by", "found_count", + "first_found", "last_found", "source", "source_payload"}) + + +def _tt_candidate_row(row, stamp): + """One TikTok corpus row → a `ut_tt_profile` candidate row. None without a handle. + + ⛔ NEVER EMITS `found_count` — the runner computes it against what is already stored, so + writing it here would reset the counter to 1 on every re-find. Same rule, same reason, as + `_candidate_row` on the Instagram side. + ⭐ THE FIELD MAP IS THE CONNECTOR'S, not a second one written here. `normalize_profile` is what + the enrich path will use too, so a corpus row and a scrape row cannot disagree about which + vendor key becomes which column. + """ + import connectors_tt as _tt + cells = _tt.normalize_profile(row if isinstance(row, dict) else {}) + if not str(cells.get("handle") or "").strip(): + return None + return {**cells, "last_found": stamp, "source": "corpus"} + + + +def _ig_discovery_series(rt, table_key, incoming, stamp, username, auto_id, counts, log): + """⭐⭐ THE FORWARD HALF (2026-08-10), Instagram only. Every candidate a run wrote also leaves + an OBSERVATION behind — see `corpus_snapshot_row`. Without it a backfill is theatre: the next + discovery run re-opens exactly the gap the backfill just closed, which is how a data defect + becomes a recurring one. + + ⚠ `ensure_ig_graph`, NOT a snapshot-only `ut_ensure`, and it is a real behaviour change worth + owning: a tenant that has never enriched gets the four canonical child databases the first time + discovery runs (four of `MAX_UT_TABLES = 40`). The alternative writes a series into a store the + profile table has no LINK to — the numbers would be recorded and unreachable, which is half a + feature wearing a whole one's clothes. + ⚠ `incoming`, not `merged`: the observation belongs to the profiles THIS RUN read, not to every + row the table happens to hold. + """ + try: + graph = ensure_ig_graph(rt, username, auto_id, profile_table=table_key) + counts["series"] = append_ig_snapshots( + rt, [corpus_snapshot_row(c, day=stamp) for c in incoming], + graph[IG_SNAPSHOTS_TABLE], log=log) + except Exception as e: # noqa: BLE001 + # The candidates already landed. A series write that fails must not throw away the rows we + # just paid the vendor for — the same posture the preset top-up takes. + log(f"[aios-auto] discover {auto_id}: corpus series write failed: " + f"{type(e).__name__}: {e}") + + +def _ig_discovery_metrics(rt, table_key, auto_id, log): + """C7 — the metric FIELDS computed off the master series. See the field runner.""" + try: + compute_metric_cells(rt, table_key) + except Exception as e: # noqa: BLE001 + log(f"[aios-auto] metric compute {table_key} failed: {type(e).__name__}: {e}") + + +#: ⭐⭐ WAVE 30 · T09 (DEBT D-129) — ONE DISCOVERY RUNNER, TWO CORPORA, AND EVERY DIFFERENCE BETWEEN +#: THEM IS A ROW IN THIS TABLE. +#: +#: `run_discover_tiktok` was written in wave 29 as a deliberate SIBLING of `run_discover_instagram` +#: — its own docstring said so and gave the reason: *"it would rewrite the one code path this +#: product bills money through"*. That was the right call with one platform's worth of evidence and +#: it stopped being right when the duplication reached ~210 lines, because a sibling does not +#: inherit fixes. The proof is already in the copy below: the TikTok runner carries the **D-116** +#: precedence rule (a corpus re-find must not overwrite an ENRICHED measurement with the corpus's +#: cheaper approximation) and the Instagram one, which is the one with paying rows in it, does not. +#: +#: ⛔ **THE COLLAPSE IS BEHAVIOUR-PRESERVING, DELIBERATELY, AND `discovery_keys` IS WHERE YOU CAN +#: SEE THAT DECISION RATHER THAN INFER IT.** Setting Instagram's to `IG_DISCOVERY_KEYS` would close +#: D-116 in one line — and it would change what a live, billed, nightly automation writes to rows +#: the owner reads, inside a refactor whose whole claim is that nothing moved. It is REPORTED +#: instead (see D-116's row), which is the standing rule for a limit that is not being removed +#: today. ⚠ The one-line fix is now visible in a table instead of buried 300 lines apart, which is +#: most of the value of doing this at all. +#: +#: ⚠ `dataset` IS A CALLABLE, not a string: `connectors_tt` imports this module at import time, so +#: the engine may only reach it from INSIDE a function. `mapper` has the same shape for the same +#: reason on the TikTok side. +#: ⚠ `handle_field` is the column the engine-added `not_in` exclusion names, and it is per platform +#: because B-20 MEASURED that Bright Data's TikTok Profiles dataset calls Instagram's `account` +#: `account_id`. The exclusion never appears as a condition row, so nothing a person types can +#: correct it. +DISCOVERY_SPECS = { + "discover_instagram": { + "noun": "profile", + "log_tag": "discover", + "platform": PLATFORM_INSTAGRAM, + "dataset": lambda: BD_DS_PROFILES, + "handle_field": "account", + "mapper": lambda row, stamp: _candidate_row(row, stamp), + "table": DISCOVER_TABLE, + "label": DISCOVER_LABEL, + "fields": CANDIDATE_FIELDS, + # ⛔ D-116 IS OPEN ON THIS SIDE. `None` = discovery overwrites whatever it re-finds. + "discovery_keys": None, + "on_write": _ig_discovery_series, + "after_write": _ig_discovery_metrics, + }, + "discover_tiktok": { + "noun": "TikTok profile", + "log_tag": "discover-tt", + "platform": PLATFORM_TIKTOK, + "dataset": lambda: _tt_module().TT_DS_PROFILES, + "handle_field": "account_id", + "mapper": lambda row, stamp: _tt_candidate_row(row, stamp), + "table": TT_PROFILE_TABLE, + "label": TT_TABLE_LABELS[TT_PROFILE_TABLE], + "fields": TT_PROFILE_FIELDS, + "discovery_keys": TT_DISCOVERY_KEYS, + # ⚠ NO series and NO metric fields on this side, and that is not an oversight: both hooks + # write into Instagram's own snapshot store, through Instagram's own vocabulary. TikTok's + # series is W29-T06's `ut_tt_post_snapshots`, which the ENRICH path owns. + "on_write": None, + "after_write": None, + }, +} + + +def _tt_module(): + """`connectors_tt`, imported LAZILY. It imports this module at module level, so a top-level + import here would close the cycle.""" + import connectors_tt as _tt + return _tt + + +def run_discovery(rt, defn, username="automation", log=print, step=_no_step, rows=None): + """Automation #3 (owner ruling R7 / D-23) and DEBT D-9: FIND handles nobody here has typed in, + on whichever corpus this automation's KIND names. Two-phase by construction — see the DISCOVERY section header. A run either STARTS a corpus - query and hands its snapshot id to the next run, or COLLECTS one a previous run started. - Every outcome is a sentence about what actually happened; none of them is a green zero. + query and hands its snapshot id to the next run, or COLLECTS one a previous run started. Every + outcome is a sentence about what actually happened; none of them is a green zero. + + ⛔ THE PLATFORM COMES FROM `defn["kind"]`, WHICH IS THE ONLY THING THAT DECIDES IT NOW. Before + wave 30 the choice was made by WHICH FUNCTION `RUNNERS` pointed at, so a fixture (or a stored + definition) with a stale kind still ran the corpus its caller had in mind. It cannot any more: + an unknown kind is refused rather than defaulted, because defaulting here means billing one + platform's corpus for another platform's search. """ + kind = str(defn.get("kind") or "") + spec = DISCOVERY_SPECS.get(kind) cfg = defn.get("config") or {} dry = bool(cfg.get("dryRun")) limit = int(cfg.get("recordsLimit") or 5) @@ -5948,6 +6451,13 @@ def run_discover_instagram(rt, defn, username="automation", log=print, step=_no_ steps = {"trigger": "ok", "find": "idle", "collect": "idle", "write": "idle"} counts = {"asked": limit, "found": 0, "new": 0, "seen_again": 0, "capped": 0, "missing_tables": 0} + if spec is None: + # ⛔ NEVER A DEFAULT. `RUNNERS` maps exactly the kinds in `DISCOVERY_SPECS` onto this + # function, so reaching here means somebody widened one of the two without the other. + steps["find"] = "blocked" + return ("error", f"this automation's kind ({kind or 'blank'}) has no profile corpus, so " + f"nothing was searched and nothing was charged", counts, [], steps) + noun = spec["noun"] if not bd_ready(): # ⛔ THE FAIL-CLOSED PATH (C4). Discovery has NO anonymous rung to drop to — the corpus is @@ -5972,7 +6482,7 @@ def run_discover_instagram(rt, defn, username="automation", log=print, step=_no_ steps["find"] = steps["write"] = "skipped" return ("partial", f"Test run — nothing saved. Would look for up to {limit} " - f"profile{'' if limit == 1 else 's'} matching " + f"{noun}{'' if limit == 1 else 's'} matching " f"{_predicate_sentence(preds, cfg.get('operator'))}, for about " f"${est['usd']}", counts, [], steps) step("Starting the search") @@ -5980,10 +6490,16 @@ def run_discover_instagram(rt, defn, username="automation", log=print, step=_no_ # appears as a condition row, and `bd_filter_start` drops it rather than risk a shape the # vendor refuses. Reported either way, because "why did this run find fewer" and "why did # this run cost the same as last night" are both questions this line answers. + # ⚠ `already_found_handles` NEEDS NO PER-PLATFORM BRANCH — it is scoped to THIS definition's + # own tables (`automation_tables`), so a TikTok automation targeting `ut_tt_profile` + # excludes exactly the handles it has itself found. Verified rather than assumed: the + # function reads `config.targetTable` plus each `create_record` action's table, and names + # no IG constant. excl = {} sid, note = bd_filter_start(preds, cfg.get("operator") or "and", limit, + dataset_id=spec["dataset"](), exclude_handles=already_found_handles(rt, defn), - applied=excl) + applied=excl, handle_field=spec["handle_field"]) if excl.get("excluded"): step(f"excluded {excl['excluded']} already-found " f"profile{'' if excl['excluded'] == 1 else 's'} at the provider") @@ -5998,7 +6514,7 @@ def run_discover_instagram(rt, defn, username="automation", log=print, step=_no_ # becomes an error run, so a `log()` placed after it never executes. The snapshot would # then exist, be billed, and have its id in neither the store nor the logs. Logging first # means the worst case is still recoverable by a human reading the Space output. - log(f"[aios-auto] discover {auto_id}: started {sid}") + log(f"[aios-auto] {spec['log_tag']} {auto_id}: started {sid}") # PERSIST BEFORE POLLING. A snapshot the vendor is already building does not stop # existing because this process dies thirty seconds later, and a lost id is a set we # paid attention to and can never collect. @@ -6008,10 +6524,10 @@ def run_discover_instagram(rt, defn, username="automation", log=print, step=_no_ waited, status, size, note = 0.0, "", 0, "" while True: # ⭐ ITEM 6 — THE ONE PLACE THE LIVE STEP HAS TO MOVE. This loop blocks for up to - # BD_FILTER_WAIT (120 s) at the vendor, and D measured that this wait IS the whole of - # "Run once now is laggy / looks stuck". A counter here is what makes a legitimate - # two-minute wait distinguishable from a hung thread; without it the rendered step reads - # the same word for both, which is a progress indicator that cannot indicate progress. + # BD_FILTER_WAIT at the vendor, and D measured that this wait IS the whole of "Run once now + # is laggy / looks stuck". A counter here is what makes a legitimate wait distinguishable + # from a hung thread; without it the rendered step reads the same word for both, which is a + # progress indicator that cannot indicate progress. step(f"Searching — {int(waited)}s of " f"{int(BD_FILTER_WAIT)}s") status, size, note = bd_filter_status(pending) @@ -6029,11 +6545,16 @@ def run_discover_instagram(rt, defn, username="automation", log=print, step=_no_ # ⚠ THE HANDOFF, AND IT IS A SUCCESSFUL OUTCOME OF A SORT. Measured build latency is ~20 # minutes; holding a worker thread that long on a free-tier container to poll would be # the wrong shape. Say plainly that it is still running and who collects it. - steps["collect"] = "partial" # ⚠ THE SENTENCE NAMES WHO COLLECTS IT, because the old one did not and that is what made # this read as "stuck": *"The next run picks up the results"* is a promise about a run # that, on a MANUAL automation, nobody had scheduled. `pending_collect_ids` makes the tick # finish it, so the promise is now kept by something rather than by the reader. + # ⭐ WAVE 27 ITEM 14 (owner) — THE BRAG IS GONE. The sentence used to explain the wait by + # naming the corpus size. It was TRUE, and it was answering a question the person had not + # asked. It was also the one place a vendor's catalogue size was quoted to a customer, so + # it would have gone stale the day the vendor grew. THE SHAPE OF THE ANSWER SURVIVES and is + # the part that mattered: the wait does not shrink when you ask for fewer records, and a + # person who does not know that reads "20 minutes for 10 rows" as a fault. mins = 0 try: since = (defn.get("state") or {}).get("pendingSince") @@ -6044,17 +6565,7 @@ def run_discover_instagram(rt, defn, username="automation", log=print, step=_no_ except Exception: # noqa: BLE001 mins = 0 been = f" — {mins} min so far" if mins else "" - # ⭐ WAVE 27 ITEM 14 (owner) — THE BRAG IS GONE. The sentence used to explain the wait by - # naming the corpus size: *"the wait is the SEARCH over 620 million profiles, not the 10 - # rows"*. It was TRUE, and it was answering a question the person had not asked. Somebody - # watching a run that has taken eighteen minutes wants to know whether it is broken and - # when it ends; how big the index is, is our fact about our supplier. Keeping it also - # meant this line was the one place a vendor's catalogue size was quoted to a customer, so - # it would have gone stale the day the vendor grew — a number nobody would think to - # re-measure because nothing depends on it. - # ⚠ THE SHAPE OF THE ANSWER SURVIVES and is the part that mattered: the wait does not - # shrink when you ask for fewer records, and a person who does not know that reads - # "20 minutes for 10 rows" as a fault. + steps["collect"] = "partial" return ("partial", f"Still searching at the provider{been}. A corpus search takes about 20 minutes " f"however few records you asked for. Nothing is lost, you are not charged twice, " @@ -6085,18 +6596,19 @@ def run_discover_instagram(rt, defn, username="automation", log=print, step=_no_ steps["collect"] = "ok" stamp = _iso() - incoming = [c for c in (_candidate_row(r, stamp) for r in rows) if c] + _map = spec["mapper"] + incoming = [c for c in (_map(r, stamp) for r in rows) if c] counts["found"] = len(incoming) # ⚠ COUNT THE ROWS WE COULD NOT USE. A corpus row with no handle cannot be a candidate, and # dropping it silently means "the vendor delivered 50 and we kept 3" reads identically to # "the vendor delivered 3". It also catches the shape where an error document comes back # down the rows path and parses as one unusable row instead of an error. counts["dropped"] = max(0, len(rows) - len(incoming)) - label = cfg.get("targetLabel") or DISCOVER_LABEL - table_key = (ut_key_for(label, cfg.get("targetTable") or DISCOVER_TABLE) if dry - else ut_ensure(rt, label, CANDIDATE_FIELDS, username, - key=cfg.get("targetTable") or DISCOVER_TABLE, - flow_tag=str(defn.get("id") or ""), lock_fields=True)) + label = cfg.get("targetLabel") or spec["label"] + table_key = (ut_key_for(label, cfg.get("targetTable") or spec["table"]) if dry + else ut_ensure(rt, label, spec["fields"], username, + key=cfg.get("targetTable") or spec["table"], + flow_tag=auto_id, lock_fields=True)) existing = dict((ut_get(rt, table_key) or {}).get("rows") or {}) # --- ⭐ WAVE 26 · C3 / owner ruling R4 — THE UPSERT KEY IS `(platform, handle)`. # @@ -6115,19 +6627,21 @@ def run_discover_instagram(rt, defn, username="automation", log=print, step=_no_ # Hence `platform`, and hence it being part of the key rather than a label beside it. _ck = candidate_key - # ⚠ A PRE-WAVE-26 ROW HAS NO `platform` CELL, and `_ck` defaults it to Instagram rather than to - # blank on purpose: every row that exists today was written by the Instagram runner, so that - # default is a FACT about the stored data, not an assumption. Blank-keyed rows would fail to - # match their own re-find and duplicate the whole table on the next run. + # ⚠ A ROW WITH NO `platform` CELL IS STAMPED WITH **THIS CORPUS'S** PLATFORM, and the default is + # per spec rather than global. On the Instagram table every legacy row came from the Instagram + # runner, so `Instagram` is a FACT about the stored data; in a `ut_tt_*` table the same blank + # means TikTok for exactly the same reason. Blank-keyed rows would fail to match their own + # re-find and duplicate the whole table on the next run. for r in existing.values(): if not str(r.get("platform") or "").strip(): - r["platform"] = PLATFORM_INSTAGRAM + r["platform"] = spec["platform"] existing2 = {rid: {**r, "_ckey": _ck(r.get("platform"), r.get("handle"))} for rid, r in existing.items()} # `found_count` is ARITHMETIC OVER WHAT IS STORED, not a value from the vendor — re-finding # a profile is the signal that it keeps matching, so the counter grows instead of # resetting. `first_found` is written only when the pair is new, so it never moves. seen = {r["_ckey"]: r for r in existing2.values()} + _dkeys = spec["discovery_keys"] for c in incoming: # "Found by" — the first finder is recorded and later finders do not overwrite them, which # is what the column means now that it is no longer part of the identity. @@ -6139,6 +6653,20 @@ def run_discover_instagram(rt, defn, username="automation", log=print, step=_no_ if prev_row: counts["seen_again"] += 1 c["found_count"] = str((_ig_int(prev_row.get("found_count")) or 0) + 1) + # ⛔⛔ DEBT D-116 — THE ENRICHED-MEASUREMENT PRECEDENCE RULE, AND IT RUNS ONLY WHERE THE + # SPEC DECLARES A `discovery_keys` SET. A discovery re-find OVERWRITES an enriched + # `followers` with the corpus number — the corpus is cheaper, rounder and older than a + # scrape, so the row silently gets WORSE every night while the automation reports + # success. The fix is a precedence rule, not a blank check: a row that has been ENRICHED + # (`enriched_at` is stamped only by the enrich path) keeps its exact measurements, and + # discovery may only fill what is genuinely empty and update its own bookkeeping. + # ⚠ The declared set is what discovery OWNS; everything else on a corpus row is a + # measurement, and an approximation must never replace an exact one. + if _dkeys and str(prev_row.get("enriched_at") or "").strip(): + for k in list(c): + if (k not in _dkeys and k != "_ckey" + and str(prev_row.get(k) or "").strip()): + c.pop(k) else: counts["new"] += 1 c["found_count"] = "1" @@ -6152,7 +6680,7 @@ def run_discover_instagram(rt, defn, username="automation", log=print, step=_no_ # `stage_`; the branch it sat in asks whether some OTHER automation had # already created the row. So the first automation to reach a handle claimed it, and every # later automation sharing that database silently had no cards — with a summary saying it - # had found them. Nothing errored, and both boards are green surfaces. + # had found them. # # ⚠ THE ORIGINAL LAW IS PRESERVED, and it is the reason this is a condition rather than an # unconditional write: a re-find must never pull a card somebody has already moved back @@ -6168,276 +6696,15 @@ def run_discover_instagram(rt, defn, username="automation", log=print, step=_no_ if not dry and not missing: ut_write_rows(rt, table_key, merged) steps["write"] = "ok" - # ⭐⭐ THE FORWARD HALF (2026-08-10). Every candidate this run wrote now also leaves an - # OBSERVATION behind — see `corpus_snapshot_row`. Without it a backfill is theatre: the - # next discovery run re-opens exactly the gap the backfill just closed, which is how a - # data defect becomes a recurring one. - # - # ⚠ `ensure_ig_graph`, NOT a snapshot-only `ut_ensure`, and it is a real behaviour change - # worth owning: a tenant that has never enriched now gets the four canonical child - # databases the first time discovery runs (four of `MAX_UT_TABLES = 40`). The alternative - # writes a series into a store the profile table has no LINK to — the numbers would be - # recorded and unreachable, which is half a feature wearing a whole one's clothes. A - # tenant running discovery is an Instagram tenant; this moves the spawn earlier, it does - # not add one. - # ⚠ `incoming`, not `merged`: the observation belongs to the profiles THIS RUN read, not - # to every row the table happens to hold. - try: - graph = ensure_ig_graph(rt, username, auto_id, profile_table=table_key) - counts["series"] = append_ig_snapshots( - rt, [corpus_snapshot_row(c, day=stamp) for c in incoming], - graph[IG_SNAPSHOTS_TABLE], log=log) - except Exception as e: # noqa: BLE001 - # The candidates already landed. A series write that fails must not throw away the - # rows we just paid the vendor for — the same posture the preset top-up takes. - log(f"[aios-auto] discover {auto_id}: corpus series write failed: " - f"{type(e).__name__}: {e}") - elif dry: - steps["write"] = "skipped" - - affected = [rid for rid, r in merged.items() - if str(r.get("handle")) in {c["handle"] for c in incoming}][:200] - est = discover_estimate(counts["found"]) - summary = (f"{counts['found']} profile{'' if counts['found'] == 1 else 's'} found — " - f"{counts['new']} new, {counts['seen_again']} seen before. About " - f"${est['usd']}") - if counts["dropped"]: - summary += (f". {counts['dropped']} result{'' if counts['dropped'] == 1 else 's'} had no " - f"username and could not be saved") - cnote = cap_note([(table_key, counts["capped"])], missing) - if cnote: - summary += f" — {cnote}" - steps["write"] = "partial" - if dry: - summary = f"Test run — nothing saved. Would have written: {summary}" - state = "partial" if (dry or counts["capped"] or missing or not counts["found"]) else "ok" - if not dry and not missing: - try: - compute_metric_cells(rt, table_key) # C7 — see the field runner - except Exception as e: # noqa: BLE001 - log(f"[aios-auto] metric compute {table_key} failed: {type(e).__name__}: {e}") - log(f"[aios-auto] discover {auto_id} -> {table_key}: {summary}") - return (state, summary, counts, affected, steps) - - -#: ⭐ W29-T04 — the discovery-run bookkeeping columns, so `run_discover_tiktok` and the D-116 merge -#: rule agree on what discovery OWNS versus what enrichment owns. Anything not in this set is a -#: MEASUREMENT, and a corpus re-find must never overwrite an enriched measurement with its own -#: cheaper approximation. -TT_DISCOVERY_KEYS = frozenset({"platform", "handle", "created_by", "found_count", - "first_found", "last_found", "source", "source_payload"}) - - -def _tt_candidate_row(row, stamp): - """One TikTok corpus row → a `ut_tt_profile` candidate row. None without a handle. - - ⛔ NEVER EMITS `found_count` — the runner computes it against what is already stored, so - writing it here would reset the counter to 1 on every re-find. Same rule, same reason, as - `_candidate_row` on the Instagram side. - ⭐ THE FIELD MAP IS THE CONNECTOR'S, not a second one written here. `normalize_profile` is what - the enrich path will use too, so a corpus row and a scrape row cannot disagree about which - vendor key becomes which column. - """ - import connectors_tt as _tt - cells = _tt.normalize_profile(row if isinstance(row, dict) else {}) - if not str(cells.get("handle") or "").strip(): - return None - return {**cells, "last_found": stamp, "source": "corpus"} - - -def run_discover_tiktok(rt, defn, username="automation", log=print, step=_no_step, rows=None): - """⭐⭐ DEBT D-9, open since wave 19 — FIND TIKTOK HANDLES NOBODY HERE HAS TYPED IN. - - Two-phase by construction, exactly like the Instagram runner: a run either STARTS a corpus - query and hands its snapshot id to the next run, or COLLECTS one a previous run started. Every - outcome is a sentence about what actually happened; none of them is a green zero. - - ⛔ IT IS A SIBLING OF `run_discover_instagram`, NOT A REFACTOR OF IT, AND THAT IS A DECISION. - The two-phase scaffolding below is duplicated. Extracting it would be architecturally right — - the `SOURCES` registry says so in its own comment — and it would rewrite the one code path this - product bills money through, which `section_bd`, `section_w27`, NC33/NC34 and four derived - sweeps all pin. R2 already chose "parallel, zero risk to the live Instagram rows" for the data; - the same reasoning applies to the runner. The extraction is booked as a PENDING line with the - Bright Data transport, which is the second duplication pointing at the same seam — two of them - is a better argument for the next wave than a half-finished refactor is for this one. - - ⛔ THE DISCOVERY HALVES BRIGHT DATA SELLS FOR TIKTOK BY KEYWORD AND BY PROFILE **404 FOR OUR - KEY** (probed: `gd_lj71gn6l68bz7y9hc`, `gd_lilwhto81z415d9mdl`). Discovery therefore routes - through the Profiles dataset's corpus filter, which is the same door Instagram uses. - """ - import connectors_tt as _tt - cfg = defn.get("config") or {} - dry = bool(cfg.get("dryRun")) - limit = int(cfg.get("recordsLimit") or 5) - preds = list(cfg.get("predicates") or []) - auto_id = str(defn.get("id") or "") - steps = {"trigger": "ok", "find": "idle", "collect": "idle", "write": "idle"} - counts = {"asked": limit, "found": 0, "new": 0, "seen_again": 0, "capped": 0, - "missing_tables": 0} - - if not bd_ready(): - # Fail-closed, and it says which credential is missing. Discovery has no free rung to drop - # to — the corpus is the vendor's. - steps["find"] = "blocked" - return ("partial", "Profile search is not set up yet — nothing was searched and nothing " - "was charged", counts, [], steps) - - pending = str((defn.get("state") or {}).get("pendingSnapshot") or "") - if not pending: - guard_err = (narrowing_refusal(preds, cfg.get("operator")) - or depth_refusal(preds, cfg.get("operator"))) - if guard_err: - steps["find"] = "blocked" - return ("error", f"the search was not started — {guard_err}", counts, [], steps) - if dry: - est = discover_estimate(limit) - steps["find"] = steps["write"] = "skipped" - return ("partial", - f"Test run — nothing saved. Would look for up to {limit} TikTok " - f"profile{'' if limit == 1 else 's'} matching " - f"{_predicate_sentence(preds, cfg.get('operator'))}, for about " - f"${est['usd']}", counts, [], steps) - step("Starting the search") - excl = {} - # ⚠ `already_found_handles` NEEDS NO TIKTOK BRANCH — it is scoped to THIS definition's own - # tables (`automation_tables`), so a TikTok automation targeting `ut_tt_profile` excludes - # exactly the handles it has itself found. Verified rather than assumed: the function reads - # `config.targetTable` plus each `create_record` action's table, and names no IG constant. - sid, note = bd_filter_start(preds, cfg.get("operator") or "and", limit, - dataset_id=_tt.TT_DS_PROFILES, - exclude_handles=already_found_handles(rt, defn), - applied=excl) - if excl.get("excluded"): - step(f"excluded {excl['excluded']} already-found " - f"profile{'' if excl['excluded'] == 1 else 's'} at the provider") - elif excl.get("dropped"): - step(f"searching without the already-found list — {excl['dropped']}") - if note: - steps["find"] = "blocked" - return ("error", f"the search was not started — {note}", counts, [], steps) - pending = sid - # LOG BEFORE PERSISTING: `set_state` can raise on an unavailable store, and a snapshot that - # exists, is billed and has its id in neither the store nor the logs is unrecoverable. - log(f"[aios-auto] discover-tt {auto_id}: started {sid}") - set_state(rt, auto_id, {"pendingSnapshot": sid, "pendingSince": _iso()}) - steps["find"] = "ok" - - waited, status, size, note = 0.0, "", 0, "" - while True: - step(f"Searching — {int(waited)}s of {int(BD_FILTER_WAIT)}s") - status, size, note = bd_filter_status(pending) - if note: - steps["collect"] = "error" - set_state(rt, auto_id, {"pendingSnapshot": None, "pendingSince": None}) - return ("error", note, counts, [], steps) - if status != "building" or waited >= BD_FILTER_WAIT: - break - time.sleep(BD_FILTER_POLL) - waited += BD_FILTER_POLL - step("Reading the profiles the search found") - - if status == "building": - mins = 0 - try: - since = (defn.get("state") or {}).get("pendingSince") - if since: - mins = max(0, int((_dt.datetime.now(_dt.timezone.utc) - - _dt.datetime.fromisoformat(str(since))).total_seconds() - // 60)) - except Exception: # noqa: BLE001 - mins = 0 - been = f" — {mins} min so far" if mins else "" - steps["collect"] = "partial" - return ("partial", - f"Still searching at the provider{been}. A corpus search takes about 20 minutes " - f"however few records you asked for. Nothing is lost, you are not charged twice, " - f"and the results are collected automatically as soon as they are ready", - counts, [], steps) - if status == "empty": - steps["collect"] = "ok" - steps["write"] = "skipped" - set_state(rt, auto_id, {"pendingSnapshot": None, "pendingSince": None}) - return ("partial", "No profiles matched. Try a shorter or more common keyword — " - "'floral' finds more accounts than 'floral design studio'", - counts, [], steps) - if status != "ready": - steps["collect"] = "error" - set_state(rt, auto_id, {"pendingSnapshot": None, "pendingSince": None}) - return ("error", f"the search ended as {status or 'unreadable'} and returned nothing", - counts, [], steps) - - rows, dnote = bd_filter_rows(pending) - if dnote: - steps["collect"] = "partial" - return ("partial", f"the search finished with {size} match" - f"{'' if size == 1 else 'es'} but {dnote}", counts, [], steps) - set_state(rt, auto_id, {"pendingSnapshot": None, "pendingSince": None}) - steps["collect"] = "ok" - - stamp = _iso() - incoming = [c for c in (_tt_candidate_row(r, stamp) for r in rows) if c] - counts["found"] = len(incoming) - # Count what we could NOT use: "the vendor delivered 50 and we kept 3" must not read the same - # as "the vendor delivered 3". - counts["dropped"] = max(0, len(rows) - len(incoming)) - label = cfg.get("targetLabel") or TT_TABLE_LABELS[TT_PROFILE_TABLE] - table_key = (ut_key_for(label, cfg.get("targetTable") or TT_PROFILE_TABLE) if dry - else ut_ensure(rt, label, TT_PROFILE_FIELDS, username, - key=cfg.get("targetTable") or TT_PROFILE_TABLE, - flow_tag=auto_id, lock_fields=True)) - existing = dict((ut_get(rt, table_key) or {}).get("rows") or {}) - # ⚠ A ROW WITH NO `platform` CELL IN A `ut_tt_*` TABLE IS STAMPED TIKTOK, not Instagram. The - # shared `candidate_key` defaults a blank to Instagram — correct for the IG table, where every - # legacy row came from that runner, and wrong here, where every row came from this one. - for r in existing.values(): - if not str(r.get("platform") or "").strip(): - r["platform"] = PLATFORM_TIKTOK - existing2 = {rid: {**r, "_ckey": candidate_key(r.get("platform"), r.get("handle"))} - for rid, r in existing.items()} - seen = {r["_ckey"]: r for r in existing2.values()} - for c in incoming: - c["created_by"] = username - c["_ckey"] = candidate_key(c.get("platform"), c["handle"]) - prev_row = seen.get(c["_ckey"]) - if prev_row and str(prev_row.get("created_by") or "").strip(): - c["created_by"] = prev_row["created_by"] - if prev_row: - counts["seen_again"] += 1 - c["found_count"] = str((_ig_int(prev_row.get("found_count")) or 0) + 1) - # ⛔⛔ DEBT D-116, AND THIS RUNNER DELIBERATELY DOES NOT INHERIT IT. On the Instagram - # side a discovery re-find OVERWRITES an enriched `followers` with the corpus number — - # the corpus is cheaper, rounder and older than a scrape, so the row silently gets - # WORSE every night while the automation reports success. The fix is a precedence rule, - # not a blank check: a row that has been ENRICHED (`enriched_at` is stamped only by the - # enrich path) keeps its exact measurements, and discovery may only fill what is - # genuinely empty and update its own bookkeeping. - # ⚠ `TT_DISCOVERY_KEYS` is what discovery OWNS; everything else on a corpus row is a - # measurement, and an approximation must never replace an exact one. - if str(prev_row.get("enriched_at") or "").strip(): - for k in list(c): - if (k not in TT_DISCOVERY_KEYS and k != "_ckey" - and str(prev_row.get(k) or "").strip()): - c.pop(k) - else: - counts["new"] += 1 - c["found_count"] = "1" - c["first_found"] = stamp - merged, mc = upsert_rows(existing2, incoming, "_ckey", cap=row_cap(table_key)) - merged = {rid: {k: v for k, v in r.items() if k != "_ckey"} - for rid, r in merged.items()} - counts["capped"] = mc["capped"] - missing = [] if dry else ut_missing(rt, table_key) - counts["missing_tables"] = len(missing) - if not dry and not missing: - ut_write_rows(rt, table_key, merged) - steps["write"] = "ok" + if spec["on_write"]: + spec["on_write"](rt, table_key, incoming, stamp, username, auto_id, counts, log) elif dry: steps["write"] = "skipped" affected = [rid for rid, r in merged.items() if str(r.get("handle")) in {c["handle"] for c in incoming}][:200] est = discover_estimate(counts["found"]) - summary = (f"{counts['found']} TikTok profile{'' if counts['found'] == 1 else 's'} found — " + summary = (f"{counts['found']} {noun}{'' if counts['found'] == 1 else 's'} found — " f"{counts['new']} new, {counts['seen_again']} seen before. About " f"${est['usd']}") if counts["dropped"]: @@ -6450,7 +6717,9 @@ def run_discover_tiktok(rt, defn, username="automation", log=print, step=_no_ste if dry: summary = f"Test run — nothing saved. Would have written: {summary}" state = "partial" if (dry or counts["capped"] or missing or not counts["found"]) else "ok" - log(f"[aios-auto] discover-tt {auto_id} -> {table_key}: {summary}") + if not dry and not missing and spec["after_write"]: + spec["after_write"](rt, table_key, auto_id, log) + log(f"[aios-auto] {spec['log_tag']} {auto_id} -> {table_key}: {summary}") return (state, summary, counts, affected, steps) @@ -6584,8 +6853,13 @@ def run_plain(rt, defn, username="automation", log=print, step=_no_step, rows=No RUNNERS = {"plain": run_plain, "scrape_db": run_scrape_db, "field_instagram": run_field_instagram, - "discover_instagram": run_discover_instagram, - "discover_tiktok": run_discover_tiktok} + # ⭐ WAVE 30 · T09 (D-129) — BOTH DISCOVERY KINDS MAP ONTO THE SAME FUNCTION. The kind is + # no longer chosen by which callable this dict holds; `run_discovery` reads it off the + # definition and looks the corpus up in `DISCOVERY_SPECS`. The two dicts are asserted + # key-for-key by the gate, because a kind in one and not the other is either an + # unroutable automation or a refused run. + "discover_instagram": run_discovery, + "discover_tiktok": run_discovery} # --------------------------------------------------------------------------------------------- @@ -6627,13 +6901,19 @@ SOURCES = { # copy, and which provider serves a capability is a routing decision `providers.py` owns. "key": "tiktok", "label": "TikTok", "vendor": "Scraper", "probe": bd_ready, - # ⚠ `capture` IS None UNTIL W29-T05 WIRES THE ENRICHMENT ACTION, and `source_status` does - # not read it — but a surface that offered a capture this source cannot do would be - # offering a button that must refuse, which is exactly what this registry's own note - # forbids. Declared as absent rather than pointed at the Instagram function. - "capture": None, + # ⭐ WAVE 30 · T08 — CAPTURE IS WIRED, and it is a LAZY reference rather than the function + # object every other row here holds. `connectors_tt` imports THIS module at module level, + # so naming `connectors_tt.pull_profile_tt` at import time is a circular import — which is + # why every existing site does `import connectors_tt` inside the function body. A + # zero-argument callable keeps the registry's shape (a reader gets a callable, not a + # string) without moving the import. + # ⚠ AND SETTING IT SWITCHES NOTHING ON. `SOURCES` is never subscripted anywhere; its one + # reader is `source_status()`, which reads `key/label/vendor/probe/discover`. An action + # kind is gated in three other places entirely (`ACTION_CATALOG`, `clean_actions`, + # `apply_actions`). This row is documentation that must not lie, not wiring. + "capture": lambda *a, **k: __import__("connectors_tt").pull_profile_tt(*a, **k), "discover": bd_filter_start, - "captureKind": None, + "captureKind": "enrich_tiktok", "discoverKind": "discover_tiktok", # No tier ladder: R6/R7 retired the free rung from enrichment entirely, and TikTok never # had one. An empty tuple is the honest statement, not a gap to fill in later. @@ -6754,10 +7034,18 @@ def graph(defn): "enters_view": f"{trg.get('table', '')} — view {trg.get('viewId') or 'unset'}", "form_submitted": str(trg.get("table") or ""), "webhook": "POST the hook URL to fire it", + # ⭐ WAVE 30 · T05 — BOTH corpus triggers, and the value is identical because the + # question is: a discovery trigger's detail line IS its filter. Without the TikTok + # key this fell to `.get(..., "")` and the trigger node rendered with a blank + # subtitle, which reads as "nothing configured" on a fully configured automation. "ig_profile_match": (_predicate_sentence(cfg.get("predicates"), cfg.get("operator"))[:80] if cfg.get("predicates") else "No filters yet — nothing to search for"), + "tiktok_profile_match": (_predicate_sentence(cfg.get("predicates"), + cfg.get("operator"))[:80] + if cfg.get("predicates") + else "No filters yet — nothing to search for"), "email": str(trg.get("query") or "")}.get(trg["key"], "") if not trg.get("configured", True): det = "Finish setting this trigger up before it can fire" @@ -6821,12 +7109,20 @@ def graph(defn): {"from": "source", "to": "capture_posts", "label": ""}, {"from": "capture_posts", "to": "capture_comments", "label": ""}, {"from": "capture_comments", "to": "write", "label": ""}] - elif defn.get("kind") == "discover_instagram": + elif defn.get("kind") in DISCOVERY_KINDS: + # ⭐⭐ WAVE 30 · T05 — WIDENED FROM `== "discover_instagram"`, AND THIS IS THE FAILURE T04 + # UNMASKS. With the seed fixed but this arm still Instagram-only, a stored TikTok search + # fell past every arm to the no-bare-`else` promise below and returned THE TRIGGER NODE + # ALONE — so no node carried `panel="find"`, and `AutomationFind` (which mounts on + # `panelKey === "find"`) could never appear. The person would have picked the trigger + # successfully and then found nowhere to type a filter: a second, stranger bug, arriving + # as the reward for fixing the first one. + _platform, _default_table, _, _ = discovery_facts(defn.get("kind")) limit = int(cfg.get("recordsLimit") or 0) est = discover_estimate(limit) pending = str((defn.get("state") or {}).get("pendingSnapshot") or "") nodes += [ - node("find", "source", "Find profiles", + node("find", "source", f"Find {_platform} profiles", _predicate_sentence(cfg.get("predicates"), cfg.get("operator"))[:60], 1, panel="find", detail=(f"Up to {limit} profiles · about ${est['usd']}" @@ -6837,7 +7133,7 @@ def graph(defn): panel="find", detail=""), node("write", "write", "Save results", - cfg.get("targetTable") or DISCOVER_TABLE, 3, panel="write", + cfg.get("targetTable") or _default_table, 3, panel="write", on=not cfg.get("dryRun"), detail=""), ] @@ -7157,6 +7453,18 @@ ACTION_CATALOG = [ "ready": True, "connector": "scraper", "detail": "Fill this record's Instagram columns from its profile, and add a point to its " "history"}, + # ⭐⭐ WAVE 30 · T08 / CONTRACT C3 — THE SECOND NETWORK, AND THIS ROW IS THE LAST SWITCH THAT + # LANDS, deliberately. `apply_actions._walk`'s kind dispatch has NO terminal `else` (measured + # from the AST, not read): an unknown kind is walked, counted, reports the run `ok`, and + # writes nothing. So a catalog entry ahead of its runner arm would ship an action that is + # addable, clickable, storable and silently inert — strictly worse than the state before it, + # where `clean_actions` refuses the kind with a sentence. + # ⚠ `connector: "scraper"` NESTS IT UNDER THE SAME BUCKET AS INSTAGRAM (R3), which is the same + # value T07 gave the trigger — one word, both menus. + {"kind": "enrich_tiktok", "label": "Enrich TikTok profile", "group": "Connected", + "ready": True, "connector": "scraper", + "detail": "Fill this record's TikTok columns from its profile, and add a point to its " + "history"}, {"kind": "send_email", "label": "Send email", "group": "Connected", "ready": False, "detail": "Needs a send scope on the Gmail connection"}, {"kind": "slack", "label": "Send Slack message", "group": "Connected", "ready": False, @@ -7405,7 +7713,10 @@ def _clean_action_config(kind, cfg, depth, seen, count): f"unique on it — it writes: " + ", ".join(sorted(clean_vals))) out["uniqueOn"] = unique return out, None - if kind == "enrich_instagram": + if kind in ENRICH_KINDS: + # ⭐ WAVE 30 · T08 — BOTH networks share this branch. See `ENRICH_KINDS`: the selection, the + # cooldown, the limit clamps and their reasons are network-agnostic, and a second copy for + # TikTok would drift until one network accepted a limit the other refused. # ⭐ WAVE 25 · C4. The switches are `field_instagram`'s, unchanged in meaning — the paid # rung, its fallback, the per-post engagement buy and the dry run — because they are the # things that decide what a run COSTS and R4 moves the capability, not the controls. @@ -7468,6 +7779,11 @@ def _clean_action_config(kind, cfg, depth, seen, count): # loses them, because refusing a retired key 400s every definition written before # this wave (D-65). # ⛔ THE FLAG THAT MULTIPLIES THE BILL BY THE POST COUNT. Off unless asked for. + # ⭐ WAVE 30 · T10 — THE TIKTOK FORCE-OFF IS GONE, because the capability it was + # standing in for now exists. T08 shipped `... and kind != "enrich_tiktok"` here + # deliberately: the key was ACCEPTED (never a 400, D-65's lesson) and answered + # honestly with `False`, because storing a `True` nothing acts on is a money switch + # claiming a capability nobody built. Both networks now read the SAME two keys. "postMetrics": bool(cfg.get("postMetrics")), # The separate Comments dataset is more granular than a post read and stays off # until the user intentionally enables it. @@ -7565,8 +7881,8 @@ def _act_row_patch(patches, table, rid, values): patches.setdefault(table, {}).setdefault(str(rid), {}).update(values) -def profile_field_key(table, named=""): - """C3/C4: WHICH COLUMN on this database carries the Instagram handle. '' when none does. +def profile_field_key(table, named="", source=PROFILE_SOURCE_IG): + """C3/C4: WHICH COLUMN on this database carries the profile handle. '' when none does. Resolution order, and the missing branch is the important one: 1. the column the action NAMES — the person picked it, so it is not a guess; @@ -7580,6 +7896,16 @@ def profile_field_key(table, named=""): reports success having enriched from the wrong column, which is worse than one that refuses: a wrong number is harder to notice than a missing one, and it would be written into the permanent history as if it had been measured. + + ⭐ WAVE 30 · T08 — `source` PARAMETERISES STEP 2 ONLY, defaulting to Instagram so every + existing caller is byte-for-byte unchanged. Step 1 (the column the person NAMED) is already + network-agnostic: a named column is a decision, not a guess, and second-guessing it against a + flag would refuse a binding somebody made on purpose. + ⛔ AND THE TWO SOURCES MUST NOT BE MERGED INTO "any profile flag". `ut_tt_profile.handle` + carries `profile: {source: "tiktok"}` and `ut_ig_profile.handle` carries `"instagram"`; a + resolver that accepted either would let a TikTok enrich step bind to an Instagram column and + spend money asking a TikTok dataset about an Instagram handle — a wrong number written into a + permanent history, which is the exact failure the no-fallback rule above exists to prevent. """ fields = (table or {}).get("fields") or [] want = str(named or "").strip() @@ -7587,17 +7913,23 @@ def profile_field_key(table, named=""): return want if any(str(f.get("key")) == want for f in fields) else "" for f in fields: p = f.get("profile") - if isinstance(p, dict) and str(p.get("source") or "") == PROFILE_SOURCE_IG: + if isinstance(p, dict) and str(p.get("source") or "") == str(source): return str(f.get("key") or "") return "" def _flow_table(defn): """The database a flow's actions and ending operate on: the automation's own target, or the - table its event trigger watches when the flow has no target of its own.""" + table its event trigger watches when the flow has no target of its own. + + ⭐ WAVE 30 · T05 — the discovery arm covers BOTH kinds. It read `== "discover_instagram"`, so a + TikTok discovery with no stored `targetTable` fell to the trigger's table — which a corpus + search does not have — and resolved to `""`. `_presets_after_write` asks this function where to + spawn the preset columns, so an empty answer there is a database that never appears. + """ cfg = (defn or {}).get("config") or {} - if defn.get("kind") == "discover_instagram": - return cfg.get("targetTable") or DISCOVER_TABLE + if defn.get("kind") in DISCOVERY_KINDS: + return cfg.get("targetTable") or discovery_facts(defn.get("kind"))[1] return cfg.get("targetTable") or ((defn.get("trigger") or {}).get("table") or "") @@ -7701,14 +8033,24 @@ def _hours_since(stamp): return max(0, int((now - t).total_seconds() // 3600)) -def _gone_key(row, handle): +def _gone_key(row, handle, platform=None): """The identity a "this account does not exist" verdict is remembered under. `(platform, handle)` — wave 26 R4's dedup key, not the handle alone, because `@x` on Instagram and `@x` on TikTok are two accounts. Blank platform means Instagram, which is what every row on a preset profile table is and what `preset_cells` stamps. + + ⭐ WAVE 30 · T08 — `platform` OVERRIDES THE ROW, and the override is what makes the key honest + on a table the row cannot speak for. The fallback above reads the row's own `platform` cell, + which the TikTok runner writes — but only AFTER the first successful pull. A hand-typed row on + an arbitrary database that a TikTok step is bound to by NAME (`profile_field_key` step 1, + which is deliberately network-blind) carries no such cell, so a TikTok "this account does not + exist" verdict would have been filed under `instagram:`. A tombstone has no expiry + (W28/R9), so that would permanently suppress an Instagram read of a DIFFERENT person's + account with the same handle — silently, and without ever spending the money that would have + shown it was wrong. The caller knows which network it asked; the row does not. """ - plat = str((row or {}).get("platform") or PLATFORM_INSTAGRAM).strip().lower() + plat = str(platform or (row or {}).get("platform") or PLATFORM_INSTAGRAM).strip().lower() return f"{plat}:{str(handle or '').strip().lstrip('@').lower()}" @@ -7816,7 +8158,7 @@ def _prime_enrich_batch(chosen, rows, profile_key, table, enrich, prefetch, return len(nodes) -def enrich_selection(rt, table_key, cfg, profile_key, today=None, gone=None): +def enrich_selection(rt, table_key, cfg, profile_key, today=None, gone=None, platform=None): """⭐ 2026-08-07 (owner ruling) — WHICH records this enrich step spends on, in order. Returns `(ordered_row_ids, note)`. The note is the honest account of what the selection did @@ -7956,7 +8298,9 @@ def enrich_selection(rt, table_key, cfg, profile_key, today=None, gone=None): # ⛔ R9 — NO EXPIRY. There is deliberately no date arithmetic here any more: a verdict is # a verdict until a human edits the cell. An `at` stamp is still STORED (it is what the # owner reads to know when we last paid to be told this), it is simply not a clock. - if isinstance((gone or {}).get(_gone_key(r, raw_handle)), dict): + # ⚠ WAVE 30 · T08 — the SAME `platform` the runner will file a new verdict under, so the + # skip and the write cannot disagree about which account a tombstone belongs to. + if isinstance((gone or {}).get(_gone_key(r, raw_handle, platform)), dict): skipped_gone.append(raw_handle) continue if cooling: @@ -8058,6 +8402,23 @@ def apply_actions(rt, defn, table_key, row_ids, username="automation", log=print # walk and written once, for the same reason every other list here is. "pendingProfiles": [], "gone": {}, "profiles": 0, "ok": 0, "blocked": 0, "notes": [], "dry": False} + # ⭐⭐ WAVE 30 · T08 — THE TIKTOK ACCUMULATOR IS ITS OWN, and the separation is the design + # decision rather than a convenience. `_enrich_flush` is not network-agnostic: it calls + # `ensure_ig_graph`, addresses the four `IG_*_TABLE` constants and writes through to the + # platform Instagram master (`ig_master.append_run`). Routing TikTok rows through it would + # append a TikTok creator's followers into Instagram's pooled history, which the metric + # FIELDS read (W29-T34) — a wrong number in a permanent series, i.e. the same failure class + # as `preset_cells`' unconditional `PLATFORM_INSTAGRAM` stamp that `tt_preset_cells` exists + # to avoid. + # ⚠ AND THE KEYS IT FLUSHES ARE PREFIXED `tt…` FOR A MEASURED REASON: `apply_actions` merges + # both flushes into ONE `counts` dict, so a shared `enriched`/`enrichBlocked` key would let a + # flow carrying BOTH an Instagram and a TikTok step report one network's numbers under the + # other's name, last writer winning, with no error anywhere. + # ⭐ W30 · D-156 — `pending` is the TikTok twin of Instagram's, and it is a RUN-level list for + # the same reason theirs is: a media snapshot belongs to a HANDLE, not to a row, so nothing + # about it needs the per-record sink that `pendingProfiles` needs. + tt = {"snaps": [], "posts": [], "psnaps": [], "comments": [], "pending": [], + "profiles": 0, "ok": 0, "blocked": 0, "notes": [], "dry": False} # The verdicts this automation already holds, read ONCE — `enrich_selection` consults them # per record and re-reading the definition per row would be a store read per record. known_gone = dict(((defn or {}).get("state") or {}).get("enrichNotFound") or {}) @@ -8133,6 +8494,28 @@ def apply_actions(rt, defn, table_key, row_ids, username="automation", log=print # A schema top-up that cannot run must not stop the enrichment: the cells that DO # have columns still land, which is strictly better than the pull being thrown away. log(f"[aios-auto] preset top-up on {table} failed: {type(exc).__name__}: {exc}") + # ⭐⭐ WAVE 30 · T08 — THE SAME TOP-UP FOR TIKTOK, AND THE SAME NARROWING: only on a table that + # is ALREADY BOUND. Separate from the block above because every constant in it is Instagram's + # (`PRESET_PROFILE_FIELDS`, `ensure_ig_graph`) and because the binding is resolved by a + # DIFFERENT question — `profile_field_key(..., source=PROFILE_SOURCE_TT)`, so a TikTok step + # cannot adopt an Instagram column and go on to ask a TikTok dataset about an Instagram handle. + # ⚠ It runs on a dry run too, exactly as the Instagram branch does: the target's own columns + # are topped up, and the related append database is NOT spawned (that is the flush's job, and + # it returns before writing anything on a dry run). + _tt_actions = _actions_of_kind(actions, "enrich_tiktok") + if _tt_actions: + _tt_named = next((str((a.get("config") or {}).get("profileField") or "").strip() + for a in _tt_actions + if str((a.get("config") or {}).get("profileField") or "").strip()), "") + _tt_bound = profile_field_key(_tbl_now, _tt_named, source=PROFILE_SOURCE_TT) + if _tt_bound: + try: + ut_ensure(rt, _tbl_now.get("label") or table, + _tt_profile_schema_for(_tt_bound), username, key=table, + flow_tag=str(defn.get("id") or ""), lock_fields=True) + except Exception as exc: # noqa: BLE001 + log(f"[aios-auto] TikTok preset top-up on {table} failed: " + f"{type(exc).__name__}: {exc}") rows = dict((ut_get(rt, table) or {}).get("rows") or {}) if not rows: return counts @@ -8239,19 +8622,23 @@ def apply_actions(rt, defn, table_key, row_ids, username="automation", log=print if not cfg.get("dryRun") else None), pending_profile=(pend_prof if not cfg.get("dryRun") else None), - prefetch=enrich_prefetch) + prefetch=enrich_prefetch, + post_groups=cfg.get("postGroups")) # ⭐ 2026-08-09 (owner: *"a Filter on our enrichment so we only get Last 12 reels / # 12 videos etc … not just last 12 but by group also"*). `config.postGroups` is # `[{"type": "video", "limit": 12}, …]`; a type nobody names is dropped. # ⛔ DEFAULT OFF — with no `postGroups` the list is returned unchanged, so this # costs nothing and changes nothing until somebody configures it. - # ⚠ IT FILTERS THE WINDOW WE ALREADY BOUGHT, it does not widen it. The profile - # read is capped at 12 posts (MEASURED: asked 40, got 12), so "12 reels AND 12 - # images" needs `bd_profile_posts` — which is built and proven (30 rows in 90 s) - # but deliberately NOT wired here: the views top-up runs INSIDE `pull_profile` - # against the posts it captured, so swapping a wider list in afterwards would - # hand back posts with no view counts. Widening belongs ahead of that step, in - # the paid path, and that is its own change. See DEBT. + # ⭐⭐ WAVE 30 · T16 — THE WINDOW IS NOW WIDENED BEFORE IT IS FILTERED, and the + # paragraph that used to sit here explaining why it could not be is gone with the + # reason. It said: *"it FILTERS the window we already bought … the views top-up + # runs INSIDE `pull_profile` against the posts it captured, so swapping a wider + # list in afterwards would hand back posts with no view counts."* Correct, and + # exactly why the widening went INTO `pull_profile` (`post_groups=`, above) rather + # than being bolted on out here — ahead of the top-up, not after it. + # ⚠ THIS LINE STAYS, and it is not redundant. A single-type ask is served by a + # native route that returns only that type; a MIXED ask has no such route, so the + # wide window comes back mixed and this is what keeps N of each. if cfg.get("postGroups"): res = {**res, "posts": select_post_groups(res.get("posts") or [], cfg.get("postGroups"))} @@ -8291,6 +8678,115 @@ def apply_actions(rt, defn, table_key, row_ids, username="automation", log=print enrich["gone"][_gone_key(row, handle_raw)] = { "at": _iso(), "handle": handle_raw, "note": _s(res.get("note"), 300)} + elif kind == "enrich_tiktok": + # ⭐⭐ WAVE 30 · T08 (carrying wave-29's dropped T05). THE SECOND NETWORK, and it + # is a sibling of the branch above rather than a flag inside it. What is genuinely + # shared is shared by CALL — `enrich_selection`, `enrich_plan`, the pace floor, + # the tombstone memory; what differs is the four things B-9 measured as + # Instagram-hardcoded, and each has a TikTok counterpart of its own name. + pkey = profile_field_key(ut_get(rt, table), cfg.get("profileField"), + source=PROFILE_SOURCE_TT) + if not pkey: + # D-79, on the network that did not exist when D-79 was written: FAIL CLOSED, + # ONCE, WITH THE REASON — and the marker is its OWN, so a flow carrying both + # steps cannot report the Instagram sentence about the TikTok one. + if "unbound" not in tt["notes"]: + tt["notes"].append("unbound") + log("[aios-auto] enrich_tiktok: no profile column on " + f"{table} — name one on the action, or mark a text column as a " + "TikTok profile") + continue + aid = str(act.get("id") or "") + if aid not in enrich_plan: + # ⚠ ONE plan dict for both networks is correct and not an oversight: it is + # keyed by ACTION id, and an action has exactly one kind. Two steps over one + # table are two budgets whichever networks they read. + chosen, sel_note = enrich_selection(rt, table, cfg, pkey, gone=known_gone, + platform=PLATFORM_TIKTOK) + enrich_plan[aid] = set(chosen) + if sel_note: + tt["notes"].append(sel_note) + if str(rid) not in enrich_plan[aid]: + continue + handle_raw = str(row.get(pkey, "") or "").strip() + if not handle_raw: + continue # nothing to enrich on this record, not an error + # ⚠ THE PACE FLOOR IS THE VENDOR'S. There is no batch prefetch on this path yet + # (`pull_profile_tt` accepts one and nothing writes it — see the mailbox), so + # every profile after the first pays it. + if tt["profiles"]: + time.sleep(PACE_SECONDS) + tt["profiles"] += 1 + import connectors_tt as _tt_conn # lazy: connectors_tt imports this module + pend_prof = [] + res = _tt_conn.pull_profile_tt( + handle_raw, log=log, + pending_profile=(pend_prof if not cfg.get("dryRun") else None), + # ⭐ W30-T10. Both INCLUDE axes default OFF (W28/R5-R7), and the same + # `config` keys the Instagram step reads — one vocabulary, two networks. + max_posts=int(cfg.get("maxPosts") or 0), + post_metrics=bool(cfg.get("postMetrics")), + comment_metrics=bool(cfg.get("commentMetrics"))) + for _p in pend_prof: + _p["table"], _p["rowId"], _p["requestedAt"] = table, str(rid), _iso() + # The deferred-profile queue is the Instagram one BY DESIGN: it is a vendor + # snapshot id waiting to be collected, and `_pending_profile_tasks` keys tasks by + # table+row, not by network. Sharing it is what makes the tick finish a TikTok + # read it has already been charged for. + enrich["pendingProfiles"].extend(pend_prof) + # ⭐⭐ W30 · D-156 — A DEFERRED MEDIA BATCH IS FILED, NOT NARRATED. Before this, + # a posts or comments scrape the vendor took too long over was reported in the + # note and collected by NOTHING: paid for, and recoverable only by a human reading + # a sentence. `connectors_tt` has already filtered these to the media corpora, so + # a profile snapshot cannot arrive here; what the engine adds is the vocabulary the + # QUEUE speaks — the `kind` (from the dataset id, which is one-to-one on TikTok) + # and the HANDLE, which the transport never knew. + # ⚠ `dryRun` queues NOTHING: a dry run buys nothing, so an entry here could only + # be a fixture leaking, and filing it would make the tick collect a snapshot that + # was never paid for. + if not cfg.get("dryRun"): + for _d in (res.get("deferredMedia") or []): + _kind = tt_metric_kind(_d.get("datasetId")) + if not _kind: + continue # not a media corpus ⇒ not this queue's business + tt["pending"].append({**_d, "kind": _kind, "requestedAt": _iso(), + "influencer": str(handle_raw).strip() + .lstrip("@").lower()}) + pulled = _iso() + if res["state"] in ("ok", "partial"): + tt["ok"] += 1 + snap_row = tt_snapshot_row(res, pulled) + if snap_row: + tt["snaps"].append(snap_row) + tt_idents, tt_metrics, tt_comments = tt_capture_rows(res, pulled) + tt["posts"].extend(tt_idents) + tt["psnaps"].extend(tt_metrics) + tt["comments"].extend(tt_comments) + cells = tt_preset_cells(res, pulled) + if cfg.get("dryRun"): + tt["dry"] = True + else: + row.update(cells) + _act_row_patch(patches, table, rid, cells) + else: + tt["blocked"] += 1 + # ⚠ TAGGED, because `run_notes` now carries BOTH networks' per-record reasons + # (the two flushes are concatenated). The summary picks ONE note to quote, so an + # untagged TikTok line could be quoted under Instagram's sentence and vice versa + # — which would undo the whole point of giving each network its own sentence. + tt["notes"].append( + f"@{handle_raw} (TikTok): {_s(res.get('note'), 300) or res['state']}") + # ⛔ AND THERE IS DELIBERATELY NO TOMBSTONE WRITER HERE, which is the opposite + # of an oversight. On the Instagram side `res["gone"]` has exactly ONE source — + # Apify answering `ACCOUNT_GONE_NOTE` (`connectors_ig`); Bright Data has no + # not-found verdict at all, and `DEFAULT_CHAINS["tt_profile"]` is deliberately + # single-provider, so nothing on this chain can say "no account exists". A + # phrase match invented against unmeasured vendor output would file a + # PERMANENT verdict (W28/R9 — no expiry) on the strength of a guess. + # ⚠ The READ side is still network-scoped above (`platform=PLATFORM_TIKTOK`) + # and that half is load-bearing today: `known_gone` is shared, Apify DOES + # write `instagram:` verdicts, and without the scoping one of those + # would silently suppress a TikTok read of a different person's account. elif kind == "find_records": found = find_records(rt, cfg.get("table"), cfg.get("cond"), int(cfg.get("limit") or 25)) @@ -8316,7 +8812,19 @@ def apply_actions(rt, defn, table_key, row_ids, username="automation", log=print if not row: continue _walk(actions, row, str(rid)) - counts.update(_enrich_flush(rt, defn, username, enrich, log)) + # ⭐⭐ WAVE 30 · T08 — TWO FLUSHES, ONE `counts`, AND THE NOTES ARE CONCATENATED RATHER THAN + # OVERWRITTEN. `RUN_NOTES_KEY` is the one key both flushes legitimately produce, so a plain + # `counts.update(a); counts.update(b)` would drop every Instagram per-record reason the moment + # a flow also carried a TikTok step — silently, and precisely on the mixed flows where a + # person most needs to know which half failed. Every other key is prefixed and cannot collide. + _ig_out = _enrich_flush(rt, defn, username, enrich, log) + _tt_out = _tt_enrich_flush(rt, defn, username, tt, log) + _flush_notes = (list(_ig_out.pop(RUN_NOTES_KEY, None) or []) + + list(_tt_out.pop(RUN_NOTES_KEY, None) or [])) + counts.update(_ig_out) + counts.update(_tt_out) + if _flush_notes: + counts[RUN_NOTES_KEY] = _flush_notes # ⭐ C5: the REALIZED numbers overwrite the walk's attempt count. `counts["created"]` was # incremented once per create the flow decided to make; what landed is what the store says, # and with `uniqueOn` on they are routinely different (a re-run of a scheduled flow matches @@ -8499,9 +9007,195 @@ def _enrich_flush(rt, defn, username, acc, log): return out +#: How many candidate columns an unbound-enrich sentence names before it stops. Three, because the +#: sentence is read in a run log line: naming forty columns is the same as naming none. +UNBOUND_HINT_MAX = 3 + + +def _unbound_hint(rt, table_key): + """⭐ WAVE 30 · T14 — the *"…which column to mark"* half of D-79's sentence. + + Returns `" — this database's text columns are Handle, Who"` or `""`. Never a binding. + + ⛔ IT SUGGESTS AND DOES NOT RESOLVE, and the distinction is the whole reason this is a string + rather than a fallback. `profile_field_key`'s own docstring refuses to guess — *"the failure + would be a run that reports success having enriched from the wrong column … a wrong number is + harder to notice than a missing one"*. That argument is about BINDING. It says nothing against + telling a person, in the sentence they are already reading, which columns are even eligible: + the enrich still refuses, nothing is written, and the human makes the choice. + ⚠ `text` ONLY, matching the writer F shipped for the flag (`ColumnMenu`'s toggle is gated on + `editType === "text"`), so the sentence cannot offer a column the editor would then refuse. + """ + fields = (ut_get(rt, table_key) or {}).get("fields") or [] + # ⚠ AND A PRESET/LOCKED COLUMN IS NOT OFFERED. `Platform` is a `text` column on every preset + # profile table and carries `automation.preset` + an `editRole`, so the field editor refuses to + # retype it — offering it would send a person to a control that says no, which is exactly the + # claim the docstring above makes and did not honour on its first draft. + names = [str(f.get("label") or f.get("key") or "").strip() for f in fields + if str(f.get("type") or "text") == "text" + and not (f.get("automation") or {}).get("preset") + and not str(f.get("editRole") or "").strip() + and str(f.get("label") or f.get("key") or "").strip()] + if not names: + return "" + shown = ", ".join(names[:UNBOUND_HINT_MAX]) + more = len(names) - UNBOUND_HINT_MAX + return (f" — this database's text columns are {shown}" + + (f" (+{more} more)" if more > 0 else "")) + + +def _tt_write_tables(rt, tag, username, snaps, posts, psnaps, comments, log=print): + """The four `ut_tt_*` tables, created-if-needed and written in ONE coalesced store update. + Returns `(inserted_by_table, capped, missing)`. + + ⭐⭐ WAVE 30 · D-156 — ONE WRITER, TWO CALLERS, AND THE SECOND CALLER IS WHY IT EXISTS. + This was the tail of `_tt_enrich_flush`, i.e. reachable only from an INLINE enrich. The + deferred collector needs exactly the same write, and the repo has already paid once for the + version where it did not: `top_up_views` lived inside `pull_profile_bd`, so it ran only when + the Posts scrape answered in time, and every DEFERRED Instagram run wrote posts with a blank + Views column. The fix there was this same shape — one function, two callers, so the inline and + deferred paths cannot answer differently — and copying the block instead would reintroduce the + class rather than the bug. + + ⭐ ONE LOOP OVER `(table, rows, key)` RATHER THAN FOUR HAND-WRITTEN BLOCKS, and the field list + comes from `TT_TABLE_FIELDS` — so a fifth `ut_tt_*` table is one tuple, and no table can be + created with a field list that disagrees with its own declaration. + ⚠ NO ROWS ⇒ NO DATABASE. Spawning `ut_tt_comments` on a collect that carried none gives a + person a database to watch never fill, which is the visible half of green-over-nothing. + """ + plan = [(TT_SNAPSHOTS_TABLE, snaps or [], "snapshot_key"), + (TT_POSTS_TABLE, posts or [], "shortcode"), + (TT_POST_SNAPSHOTS_TABLE, psnaps or [], "post_snapshot_key"), + (TT_COMMENTS_TABLE, comments or [], "comment_key")] + written, capped, missing = {}, 0, [] + inserted = {} + for table_key, rows_in, key_field in plan: + if not rows_in: + continue + real_key = ut_ensure(rt, TT_TABLE_LABELS[table_key], TT_TABLE_FIELDS[table_key], username, + key=table_key, flow_tag=tag, lock_fields=True) + missing.extend(ut_missing(rt, real_key)) + merged, counts_ = upsert_rows(dict((ut_get(rt, real_key) or {}).get("rows") or {}), + rows_in, key_field, cap=row_cap(real_key)) + written[real_key] = merged + inserted[table_key] = counts_["inserted"] + capped += counts_["capped"] + + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + for k, rws in written.items(): + if cur.get(k) is not None: + cur[k]["rows"] = rws + _refresh_relations_inplace(cur, log=log) + return cur + + if written: + rt.update(UT_STORE_KEY, _up, flush="sync") # ONE coalesced update for every table + return inserted, capped, missing + + +def _tt_enrich_flush(rt, defn, username, acc, log): + """⭐⭐ WAVE 30 · T08 — the TikTok enrich action's append table, written ONCE for the whole run. + + The twin of `_enrich_flush` and deliberately NOT a call into it. That function is bound to + Instagram at four points — `ensure_ig_graph`, the four `IG_*_TABLE` keys, `PRESET_*` and the + write-through to the platform Instagram master — and the last of those is the one that would + look harmless: `ig_master.append_run` is what the metric FIELDS read (W29-T34), so a TikTok + creator's followers appended there would become an Instagram number in a pooled history that + no later run could tell apart. + + ⛔ EVERY KEY IT RETURNS IS PREFIXED `tt…`. `apply_actions` merges both flushes into one + `counts`, so `enriched`/`enrichBlocked` would be one name for two networks' numbers and a + mixed flow would report whichever flushed last. + + ⭐ WAVE 30 · T10 — FOUR TABLES NOW, AND EACH IS CREATED ONLY WHEN IT HAS A ROW. Spawning + `ut_tt_posts` on a run that captured no posts gives a person a database to watch never fill, + which is the visible half of green-over-nothing and the same complaint that produced + `discover_default_table` (*"a SECOND, empty database"*). So the create rides the rows. + """ + if "unbound" in (acc.get("notes") or []): + # D-79 on the second network: a COUNT, so `run_now`'s existing non-zero merge carries it + # and the run entry itself records that the step could not run. Its own key, so the + # summary can name TikTok rather than borrowing Instagram's sentence. + return {"ttEnrichUnbound": 1} + if not acc.get("profiles"): + # The selection's note outlives a run that read nothing — the same reason as the IG side: + # "every candidate was skipped" is exactly the run whose silence needs explaining. + notes = list(acc.get("notes") or []) + return {RUN_NOTES_KEY: notes} if notes else {} + out = {"ttEnriched": acc["ok"], "ttEnrichBlocked": acc["blocked"]} + if acc.get("notes"): + out[RUN_NOTES_KEY] = list(acc["notes"]) + # ⭐⭐ W30 · D-156 — QUEUED BEFORE THE DRY-RUN AND EMPTY-ROWS RETURNS BELOW, and the order is + # the point. A run whose media batch DEFERRED has no posts and no comments to write, so it + # takes the `not (snaps or posts or ...)` exit — the exact run that owns a paid snapshot id. + # Filing after that return would have collected nothing, forever, which is the shape the + # inline-vs-deferred split keeps producing. + # ⚠ Its own count key (`tt…`), like every other key here: `apply_actions` merges both + # networks' flushes into ONE dict, so sharing Instagram's name would let a mixed flow report + # one network's pending batches under the other's. + if not acc.get("dry"): + queued = queue_pending_metric_snapshots(rt, str(defn.get("id") or ""), + acc.get("pending") or []) + if queued: + out["ttEnrichMetricBatchesPending"] = queued + if acc.get("dry") or not (acc.get("snaps") or acc.get("posts") or acc.get("psnaps") + or acc.get("comments")): + # A dry run resolves nothing and writes nothing — not even `ut_ensure`, which CREATES. + if acc.get("dry"): + out["ttEnrichDryRun"] = acc["profiles"] + return out + tag = str(defn.get("id") or "") + inserted, capped, missing = _tt_write_tables( + rt, tag, username, acc.get("snaps"), acc.get("posts"), acc.get("psnaps"), + acc.get("comments"), log) + # LOUD, never silent (D-11): a full append table means the SERIES has stopped growing, which + # is the failure a chart cannot show you. + if capped: + out["ttEnrichCapped"] = capped + log(f"[aios-auto] enrich_tiktok: {capped} row(s) refused by a table's row cap") + if missing: + out["ttEnrichMissingTables"] = len(missing) + log(f"[aios-auto] enrich_tiktok: could not create {', '.join(missing)}") + out["ttEnrichPoints"] = len(acc.get("snaps") or []) + if inserted.get(TT_POSTS_TABLE): + out["ttEnrichPosts"] = inserted[TT_POSTS_TABLE] + if inserted.get(TT_COMMENTS_TABLE): + out["ttEnrichComments"] = inserted[TT_COMMENTS_TABLE] + # ⛔ AND NO PLATFORM-MASTER WRITE-THROUGH, which is a deliberate absence rather than a missing + # line. `ig_master` is Instagram's pooled history and there is no TikTok equivalent yet; the + # tenant's own `ut_tt_snapshots` is the complete record today, and inventing a second store + # for a series nobody reads would be the fork this function exists to avoid. + return out + + _PENDING_METRIC_DATASETS = frozenset((BD_DS_POSTS, BD_DS_REELS, BD_DS_COMMENTS)) _PENDING_METRIC_KINDS = frozenset(("posts", "comments")) +#: ⭐⭐ WAVE 30 · D-156 — THE TIKTOK HALF OF THE METRIC QUEUE, BUILT LAZILY FROM `connectors_tt`'s +#: OWN CONSTANTS. Re-declaring the two ids here would be a second copy of a vendor identifier that +#: nothing compares — the drift `MAX_UT_ROWS` demonstrated at 12x — so this reads them from the one +#: module that owns them, through `_tt_module()` because `connectors_tt` imports THIS module. +#: ⚠ It is a MAP rather than a set because on TikTok one dataset is exactly one kind, which is why +#: this network needs no `_tag_metric_deferrals` twin: the id the transport already recorded says +#: whether a batch is posts or comments, so nothing downstream has to be told twice. +_TT_METRIC_KIND_BY_DATASET = None + + +def tt_metric_kind(dataset_id): + """`"posts"` / `"comments"` for a TikTok media dataset id; `""` for anything else. + + ⛔ THE `""` IS LOAD-BEARING AND IS THE PLATFORM TEST. `collect_pending_metric_snapshots` + branches on it, so a dataset this map does not know keeps Instagram's mappers — which is the + safe direction, because Instagram's are what every stored pre-wave-30 task needs. + """ + global _TT_METRIC_KIND_BY_DATASET + if _TT_METRIC_KIND_BY_DATASET is None: + _tt = _tt_module() + _TT_METRIC_KIND_BY_DATASET = {str(_tt.TT_DS_POSTS): "posts", + str(_tt.TT_DS_COMMENTS): "comments"} + return _TT_METRIC_KIND_BY_DATASET.get(str(dataset_id or ""), "") + def _pending_metric_tasks(defn): """Read validated, deduplicated paid metric snapshots from automation continuation state. @@ -8520,7 +9214,12 @@ def _pending_metric_tasks(defn): dataset = str(item.get("datasetId") or "").strip() kind = str(item.get("kind") or "").strip() handle = str(item.get("influencer") or "").strip().lstrip("@").lower() - if (not sid.startswith("sd_") or dataset not in _PENDING_METRIC_DATASETS + # ⭐ W30 · D-156 — BOTH NETWORKS' MEDIA CORPORA ARE COLLECTABLE NOW. The membership test + # stays a WHITELIST (a snapshot id is a vendor capability that has already been paid for; + # accepting an unknown dataset would send our key at a corpus no mapper here can read), + # and TikTok's half is asked of the map that owns it rather than listed again. + if (not sid.startswith("sd_") + or (dataset not in _PENDING_METRIC_DATASETS and not tt_metric_kind(dataset)) or kind not in _PENDING_METRIC_KINDS or not handle): continue key = (sid, dataset, kind, handle) @@ -8901,6 +9600,10 @@ def collect_pending_metric_snapshots(rt, defn, username="automation", log=print, return ("ok", "No pending post-engagement snapshots.", {}, [], {"capture_posts": "idle"}) step(f"Collecting {len(tasks)} post-engagement batch{'' if len(tasks) == 1 else 'es'}") remaining, idents, snapshots, comments = [], [], [], [] + # ⭐ W30 · D-156 — TikTok's rows accumulate SEPARATELY and are written by TikTok's own writer. + # One queue can hold both networks' snapshots (they are keyed by dataset), but the two write + # paths address different tables and neither may touch the other's. + tt_collected = {"posts": [], "psnaps": [], "comments": []} ready, waiting, closed, run_notes = 0, 0, 0, [] for task in tasks: # ⭐⭐ 2026-08-09 — ASK THE STATUS DOCUMENT, NOT THE ROWS. `building` used to be @@ -8927,6 +9630,35 @@ def collect_pending_metric_snapshots(rt, defn, username="automation", log=print, continue ready += 1 pulled = _iso() + # ⭐⭐ WAVE 30 · D-156 — THE TIKTOK ARM. Before this, `_PENDING_METRIC_DATASETS` was + # Instagram's three, so a TikTok media batch the vendor deferred was paid for and + # collected by NOTHING — the note carried the snapshot id and a human was the only + # collector. It routes through `connectors_tt`'s own mappers and `tt_capture_rows`, + # never Instagram's, because every one of those stamps `PLATFORM_INSTAGRAM`. + # ⛔ TWO THINGS THE INSTAGRAM ARM DOES THAT THIS ONE MUST NOT, both deliberate: + # * NO `top_up_views`. TikTok's `play_count` arrives inline on the posts row, so there + # is no views capability to route to; calling it would buy an Instagram permalink. + # * NO `ig_master.append_run`. There is no TikTok master by design (`_tt_enrich_flush`), + # and a write-through would put a TikTok creator into Instagram's pooled history, + # which the metric FIELDS read — a wrong number in a permanent series. + if tt_metric_kind(task["datasetId"]): + _tt = _tt_module() + who = task["influencer"] + if task["kind"] == "posts": + mapped = [m for m in (_tt.normalize_post(r) for r in rows) if m] + for m in mapped: + # The snapshot was bought from THIS profile's own permalinks, so the backlink + # is a fact of the call even when a vendor row omits the author field. + m.setdefault("influencer_key", who) + res_tt = {"profile": {"handle": who}, "posts": mapped, "comments": []} + else: + mapped = [m for m in (_tt.normalize_comment(r) for r in rows) if m] + res_tt = {"profile": {"handle": who}, "posts": [], "comments": mapped} + tt_idents, tt_metrics, tt_comments = tt_capture_rows(res_tt, pulled) + tt_collected["posts"].extend(tt_idents) + tt_collected["psnaps"].extend(tt_metrics) + tt_collected["comments"].extend(tt_comments) + continue if task["kind"] == "posts": mapped_posts = [] for raw in rows: @@ -8973,6 +9705,23 @@ def collect_pending_metric_snapshots(rt, defn, username="automation", log=print, "metricBatchesEmpty": closed, "postEngagementSnapshots": written["snapshots"], "commentsCollected": written["comments"]} + # ⭐ W30 · D-156 — the TikTok write, through the SAME function the inline enrich uses, so a + # collected batch and an inline one cannot land differently. Its counts carry the `tt` prefix + # for the reason every other TikTok count does. + if any(tt_collected.values()): + tt_inserted, tt_capped, tt_missing = _tt_write_tables( + rt, str(defn.get("id") or ""), username, [], tt_collected["posts"], + tt_collected["psnaps"], tt_collected["comments"], log) + for key, name in ((TT_POSTS_TABLE, "ttPostsCollected"), + (TT_POST_SNAPSHOTS_TABLE, "ttPostSnapshotsCollected"), + (TT_COMMENTS_TABLE, "ttCommentsCollected")): + if tt_inserted.get(key): + counts[name] = tt_inserted[key] + if tt_capped: + counts["ttCollectCapped"] = tt_capped + log(f"[aios-auto] collect: {tt_capped} TikTok row(s) refused by a table's row cap") + if tt_missing: + log(f"[aios-auto] collect: could not create {', '.join(tt_missing)}") # ⚠ THE RUNNER CONTRACT STAYS A 5-TUPLE and the notes ride in `counts` under the reserved # `RUN_NOTES_KEY`, which `run_now` pops. Widening the tuple for one runner would make four # other call sites disagree about the shape of a run — and `_commit_run` already drops @@ -10102,16 +10851,35 @@ TRIGGER_GROUP_OF = { #: ⚠ THESE KEYS ARE GROUPING HANDLES FOR THE PICKER, NOT connector-directory slugs, and the two #: genuinely differ: the directory's OAuth row for Gmail is `google` (the provider), while a #: person choosing a trigger is picking *Gmail* (the product). A client that joined this key -#: against `/connectors/directory` would match `scraper` and `tiktok` and miss `gmail` — so it +#: against `/connectors/directory` would match `scraper` and `webhooks` and miss `gmail` — so it #: must group by it and render `label`, never look it up. Said here because the miss would be #: silent and partial, which is the worst shape. +#: (⚠ that example USED to read "`scraper` and `tiktok`" — R3 retired `tiktok` as a handle, and +#: the sentence is corrected here rather than left to rot into a lie about a key that is gone.) TRIGGER_CONNECTOR = { "email": {"key": "gmail", "label": "Gmail"}, "webhook": {"key": "webhooks", "label": "Webhooks"}, - # R9: the Scraper ships with ONE built trigger and one faded one. + # ⭐ WAVE 30 · R3 — ONE "SCRAPER" BUCKET, AND IT HOLDS BOTH PLATFORMS. + # The owner, verbatim and for the third wave running: *"I say this multiple times already the + # damn Tiktok and Instagram belongs in the same bucket when creating the automation its under + # Scraper … Only when I click 'Scraper' under each automation trigger and actions would I see + # the option to choose either Instagram OR TikTok. That's it."* + # + # ⛔ THIS SUPERSEDES WAVE 29's RULE, and the old rule was not a typo — it was an argument: + # *"TikTok is its own connector, not the Scraper's … because the sub-group answers WHICH + # PRODUCT and never HOW BUILT."* Coherent, and not what was asked for. Instagram and TikTok are + # two PRODUCTS of one CAPABILITY (a social scraper bought from one vendor); a person opening + # this picker is choosing the capability first and the platform second. `verify_automation` + # asserted the old rule as an assertion AND as prose — a shipped gate forbidding the owner's + # ruling is most of why this complaint survived two waves — so it is INVERTED in this same + # change, comment included. + # + # ⚠ The Scraper sub-group now holds THREE rows: two built (Instagram, TikTok) and one faded + # (the page-change trigger). No display ORDER is emitted here — the client groups on `key` and + # owns its own ordering (contract C1). `tiktok` ceases to exist as a grouping handle. "ig_profile_match": {"key": "scraper", "label": "Scraper"}, "web_page_changed": {"key": "scraper", "label": "Scraper"}, - "tiktok_profile_match": {"key": "tiktok", "label": "TikTok"}, + "tiktok_profile_match": {"key": "scraper", "label": "Scraper"}, } #: Triggers that watch a database and therefore need one named before they can fire. TRIGGER_TABLE_KEYS = ("event_field", "record_updated", "record_created", "enters_view", @@ -10845,6 +11613,8 @@ def compose_sentence(defn): head = f"When a record in {trg.get('table')} is updated" elif trg.get("key") == "ig_profile_match": head = "When an Instagram profile fits the criteria" + elif trg.get("key") == "tiktok_profile_match": + head = "When a TikTok profile fits the criteria" elif trg.get("key") == "record_created": head = f"When a record is created in {trg.get('table')}" elif trg.get("key") == "webhook": @@ -10862,9 +11632,18 @@ def compose_sentence(defn): # ⚠ NO RUNG CLAUSE (R5). There is one way to capture a profile now, so ", exact counts # first" / ", anonymous only" described a choice that no longer exists. body = f"capture Instagram profiles for {cfg.get('targetTable') or 'the database'}" - elif kind == "discover_instagram": - body = (f"search Instagram for up to {cfg.get('recordsLimit') or 0} profiles into " - f"{cfg.get('targetTable') or DISCOVER_TABLE}") + elif kind in DISCOVERY_KINDS: + # ⭐⭐ WAVE 30 · T05 — the arm covers both kinds, and the network is NAMED from the kind + # rather than hard-coded into the sentence. Instagram-only, a TikTok search fell into the + # `plain` branch below and introduced itself as *"do nothing yet — this automation has no + # actions"*: a flatly false sentence, on the one surface whose stated promise is that it + # cannot lie about what runs. (The `plain` branch's own comment records the mirror-image + # incident — it USED to be this arm, and described every plain automation as an Instagram + # search for 0 profiles. The same two branches have now mis-described each other's + # automations in both directions, which is why neither may be a fallthrough.) + _dplatform, _dtable, _, _ = discovery_facts(kind) + body = (f"search {_dplatform} for up to {cfg.get('recordsLimit') or 0} profiles into " + f"{cfg.get('targetTable') or _dtable}") else: # ⭐ WAVE 24 — `plain` describes itself from its FLOW, because the flow is all it has. # ⛔ THIS BRANCH USED TO BE `discover_instagram`'s, so before the arm above existed every @@ -10985,7 +11764,24 @@ def run_now(rt, tenant, auto_id, username="automation", log=print, rows=None): state = "partial" if state != "error" else state summary += (" — the Instagram step did not run: this database has no profile " "column. Name one on the step, or mark a text column as the " - "Instagram profile") + "Instagram profile" + + _unbound_hint(rt, _flow_table(defn))) + if a_counts.get("ttEnrichUnbound"): + # ⛔ WAVE 30 · T08 — ITS OWN SENTENCE, not the one above with a word swapped by a + # variable. A flow may carry BOTH steps, and the fix a person has to apply is + # per-column: naming an Instagram profile column does nothing for a TikTok step, + # so a single sentence covering "the enrich step" would send them to the wrong + # place half the time. Both may appear on one run, which is correct. + state = "partial" if state != "error" else state + summary += (" — the TikTok step did not run: this database has no TikTok profile " + "column. Name one on the step, or mark a text column as a TikTok " + "profile" + + _unbound_hint(rt, _flow_table(defn))) + if a_counts.get("ttEnrichBlocked"): + state = "partial" if state != "error" else state + tt_note = next((n for n in run_notes if "(TikTok): " in n), "") + summary += (f" — {int(a_counts['ttEnrichBlocked'])} TikTok profile read(s) were " + "blocked" + (f". {_s(tt_note, 220)}" if tt_note else "")) if a_counts.get("enrichProfileBatchesPending"): # ⭐ The paid profile the vendor is still building. Said out loud so a run that # looks like a failure is read as the handoff it is — the tick finishes it. @@ -10999,7 +11795,10 @@ def run_now(rt, tenant, auto_id, username="automation", log=print, rows=None): # read(s) were blocked" is the exact string the owner read three mornings running # before asking "wtf is going on"; it names a quantity and withholds the one # thing that would let anybody act. The first note is the vendor's own words. - blocked_note = next((n for n in run_notes if ": " in n), "") + # ⚠ ...and the Instagram selector SKIPS the tagged TikTok lines for the same + # reason. Two sentences quoting each other's vendor reason is worse than one. + blocked_note = next((n for n in run_notes + if ": " in n and "(TikTok): " not in n), "") summary += (f" — {int(a_counts['enrichBlocked'])} profile read(s) were blocked" + (f". {_s(blocked_note, 220)}" if blocked_note else "")) # ⭐ WAVE 25 · C5 — A FULL TARGET IS A `partial` RUN THAT SAYS SO. D-11 made this the @@ -11056,7 +11855,19 @@ def pending_collect_ids(rt): continue if (d.get("trigger") or {}).get("paused"): continue - pending_discovery = (d.get("kind") == "discover_instagram" + # ⭐⭐ WAVE 30 · T05 — BOTH discovery kinds, and this one is a MONEY defect rather than a + # cosmetic one. It tested `== "discover_instagram"`, so a TikTok corpus search stored its + # `pendingSnapshot`, told the person *"The next run picks up the results"*, and was then + # never returned by this function — the tick collected nothing, forever. That is EXACTLY + # the live incident quoted in the docstring above (*"why is the SMALL 10 records test + # search taking forever, its not populating"*), reproduced for the second platform by the + # wave that added it: a result the tenant has already been charged for, stranded. + # ⚠ NOT ON THE SCOUT'S LIST OF FIVE. Found by reading this function for a different + # ticket, which is the argument for `DISCOVERY_KINDS` in one line — the sites that test a + # kind string are not enumerable by memory, and this one is three thousand lines from the + # others. Pressing Run again does still collect (the runner's own branch reads the same + # field), so the money was recoverable BY HAND and only ever silently lost on a schedule. + pending_discovery = (d.get("kind") in DISCOVERY_KINDS and str((d.get("state") or {}).get("pendingSnapshot") or "").strip()) # ⭐ 2026-08-09 — PROFILE handoffs join the other two. Without this line the profile # deferral would be stored and never collected, which is the same defect it fixes wearing