Automated sync from GitHub

.gitattributes

@@ -1,8 +1,8 @@
+*.ttf filter=lfs diff=lfs merge=lfs -text
+*.otf filter=lfs diff=lfs merge=lfs -text
 *.webp filter=lfs diff=lfs merge=lfs -text
 *.mp4 filter=lfs diff=lfs merge=lfs -text
 *.mp3 filter=lfs diff=lfs merge=lfs -text
 *.jpg filter=lfs diff=lfs merge=lfs -text
 *.jpeg filter=lfs diff=lfs merge=lfs -text
 *.png filter=lfs diff=lfs merge=lfs -text
-*.ttf filter=lfs diff=lfs merge=lfs -text
-*.otf filter=lfs diff=lfs merge=lfs -text

app/autopost.py

@@ -0,0 +1,81 @@
+"""/autopost: the posts of a whole series - "Episodes 01 to 20", "Episodes 21 to 40", ... - each with a download button.
+
+The ranges are planned here (pure functions, easy to test); the handler in handlers/autopost.py sends them.
+"""
+from __future__ import annotations
+
+PLACEHOLDER_LINK = "https://t.me/English"  # every button starts with this link; the real ones are asked for afterwards
+MAX_POSTS = 200  # the most one /autopost run may post
+MAX_EPISODES = 100_000
+
+USAGE = (
+    "Usage: <code>/autopost &lt;total episodes&gt; &lt;interval&gt;</code>\n"
+    "Example: <code>/autopost 114 20</code> posts Episodes 01 to 20, 21 to 40 ... up to 114, each with a download "
+    "button, and then asks you for the real link of every button."
+)
+
+
+def parse_autopost_args(raw: str) -> tuple:
+    """'114 20' -> (114, 20). ValueError (with a message to show) when it can't be used."""
+    parts = (raw or "").replace(",", " ").split()
+    if len(parts) != 2 or not all(p.isdigit() for p in parts):
+        raise ValueError("I need two numbers: the total number of episodes and the interval.")
+    total, interval = int(parts[0]), int(parts[1])
+    if total < 1 or interval < 1:
+        raise ValueError("Both numbers must be at least 1.")
+    if total > MAX_EPISODES:
+        raise ValueError(f"The total can be at most {MAX_EPISODES}.")
+    return total, interval
+
+
+def plan_ranges(total: int, interval: int) -> list:
+    """[(first, last), ...] covering 1..total in steps of `interval`.
+
+    The last, shorter range gets its own post when it is longer than half an interval (114 with 20 -> 101 to 114);
+    when it is half an interval or less it is added to the post before it (102 with 20 -> 81 to 102)."""
+    if total < 1 or interval < 1:
+        raise ValueError("total and interval must be at least 1")
+    ranges, start = [], 1
+    while start <= total:
+        end = min(start + interval - 1, total)
+        ranges.append((start, end))
+        start = end + 1
+    if len(ranges) >= 2:
+        a, b = ranges[-1]
+        size = b - a + 1
+        if size < interval and size * 2 <= interval:
+            ranges.pop()
+            ranges[-1] = (ranges[-1][0], b)
+    return ranges
+
+
+def pad(n: int) -> str:
+    return f"{n:02d}"
+
+
+def span(a: int, b: int) -> str:
+    return f"{pad(a)} to {pad(b)}"
+
+
+def title_of(a: int, b: int) -> str:
+    return f"Episodes {span(a, b)}" if a != b else f"Episode {pad(a)}"
+
+
+def post_text(a: int, b: int) -> str:
+    return f"{title_of(a, b)}\n\nClick the button below to download 👇"
+
+
+def button_label(a: int, b: int) -> str:
+    return f"Download {title_of(a, b)}"
+
+
+def post_buttons(a: int, b: int, link: str = PLACEHOLDER_LINK) -> list:
+    return [[{"t": button_label(a, b), "u": link}]]
+
+
+def ranges_summary(ranges: list, show: int = 8) -> str:
+    """'01–20 · 21–40 · 41–60 · … · 101–114' (the middle is left out of long lists)."""
+    parts = [f"{pad(a)}–{pad(b)}" if a != b else pad(a) for a, b in ranges]
+    if len(parts) > show:
+        parts = parts[: show - 2] + ["…"] + parts[-2:]
+    return " · ".join(parts)

app/buttons.py

@@ -64,6 +64,12 @@ def missing_links(rows: list) -> int:
     return sum(1 for r in rows or [] for b in r if not b.get("u") and not b.get("raw"))
 
 
+def link_positions(rows: list) -> list:
+    """(row, column) of every button that has a link of its own. Other bots' buttons (reactions...) are left out:
+    they can't be edited."""
+    return [(r, c) for r, row in enumerate(rows or []) for c, b in enumerate(row) if b.get("u") and not b.get("raw")]
+
+
 def valid(rows: list, r: int, c: int) -> bool:
     return 0 <= r < len(rows) and 0 <= c < len(rows[r])
 

app/channelref.py

@@ -0,0 +1,119 @@
+"""Find a channel from what a person types: @username, t.me link, numeric id (-100...), or an invite link.
+
+Used by /shift. The bot can only talk to a channel it can see: registered channels (the database has their access
+details), channels it has met in updates, public channels by @username, and channels it is a member of (invite link).
+"""
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from typing import Optional
+
+from telethon import functions, types
+
+from .common import Ctx, UserError
+
+_REF_RX = re.compile(r"(?i)^(?:https?://)?(?:t\.me|telegram\.me)/(?:c/(\d+)|([A-Za-z0-9_]{3,32}))(?:/\d+)?/?$")
+_INVITE_RX = re.compile(r"(?i)^(?:https?://)?(?:t\.me|telegram\.me|telegram\.dog)/(?:\+|joinchat/)([A-Za-z0-9_-]{6,})/?$")
+_TG_INVITE_RX = re.compile(r"(?i)^tg://join\?invite=([A-Za-z0-9_-]{6,})$")
+
+
+def parse_channel_ref(text: str):
+    """@name / t.me link / numeric id -> username (str) or channel id (int); None if unrecognised."""
+    t = (text or "").strip()
+    m = _REF_RX.match(t)
+    if m:
+        return int(m.group(1)) if m.group(1) else m.group(2)
+    t = t.lstrip("@")
+    if re.fullmatch(r"-?\d+", t):
+        s = str(abs(int(t)))
+        return int(s[3:]) if s.startswith("100") and len(s) > 10 else int(s)
+    if re.fullmatch(r"[A-Za-z0-9_]{3,32}", t):
+        return t
+    return None
+
+
+def classify_ref(text: str) -> tuple:
+    """-> ("invite", hash) | ("username", name) | ("id", int) | (None, None)"""
+    t = (text or "").strip()
+    m = _INVITE_RX.match(t) or _TG_INVITE_RX.match(t)
+    if m:
+        return "invite", m.group(1)
+    ref = parse_channel_ref(t)
+    if isinstance(ref, int):
+        return "id", ref
+    if isinstance(ref, str):
+        return "username", ref
+    return None, None
+
+
+@dataclass
+class ChannelRef:
+    """A channel as the bot can reach it."""
+
+    id: int
+    access_hash: int
+    title: str
+    username: Optional[str]
+    registered: bool  # the bot's database knows it (it can be used with /new, /posts ...)
+
+
+def _from_entity(ent, registered: bool) -> ChannelRef:
+    if not isinstance(ent, types.Channel):
+        raise UserError("That is not a channel (it looks like a user, bot or group). Only channels can be used here.")
+    if not ent.broadcast:
+        raise UserError("That is a group, not a channel. Only channels can be used here.")
+    if not ent.access_hash or getattr(ent, "min", False):
+        raise UserError(
+            "I can see that channel, but Telegram has not given me its access details. Use its @username, or add the "
+            "bot to the channel as an admin and post something there first."
+        )
+    return ChannelRef(ent.id, ent.access_hash, ent.title or "", ent.username, registered)
+
+
+async def _entity(client, what):
+    try:
+        return await client.get_entity(what)
+    except Exception:
+        return None
+
+
+async def resolve_channel(ctx: Ctx, text: str) -> ChannelRef:
+    """The channel `text` points at, or a UserError that says what to do."""
+    kind, value = classify_ref(text)
+    if kind is None:
+        raise UserError(
+            f"“{text}” is not a channel I understand. Use an @username, a t.me link, the channel id (-100…) or an invite link."
+        )
+    db, client = ctx.db, ctx.client
+    if kind == "username":
+        for ch in await db.list_channels(active_only=False):
+            if ch.username and ch.username.lower() == value.lower():
+                return ChannelRef(ch.id, ch.access_hash, ch.title, ch.username, ch.active)
+        ent = await _entity(client, value)
+        if ent is None:
+            raise UserError(f"I couldn't find a channel called @{value}. Check the name, or add the bot to it as an admin.")
+    elif kind == "id":
+        ch = await db.get_channel(value)
+        if ch is not None:
+            return ChannelRef(ch.id, ch.access_hash, ch.title, ch.username, ch.active)
+        ent = ctx.seen_channels.get(value) or await _entity(client, types.PeerChannel(value))
+        if ent is None:
+            raise UserError(
+                f"I don't know a channel with the id {value}. A bot can only use an id for channels it is part of; "
+                "use the @username instead, or add the bot to the channel as an admin."
+            )
+    else:  # invite link: only tells which channel it is; the bot cannot join by itself
+        try:
+            info = await client(functions.messages.CheckChatInviteRequest(hash=value))
+        except Exception as e:
+            raise UserError(f"That invite link doesn't work ({type(e).__name__}).")
+        chat = getattr(info, "chat", None)
+        if chat is None:
+            raise UserError(
+                "The bot is not a member of that channel, so it cannot use that invite link. Add the bot to the channel "
+                "as an admin first (a bot cannot join by itself)."
+            )
+        ent = chat
+    known = await db.get_channel(getattr(ent, "id", 0))
+    return _from_entity(ent, known is not None and known.active)

app/common.py

@@ -35,6 +35,9 @@ class Ctx:
     lock: asyncio.Lock = field(default_factory=asyncio.Lock)  # one long job (replace / repost) at a time
     userbot: Any = None  # app.userbot.Userbot or None
     reporter: Any = None  # app.errorlog.ErrorReporter or None
+    expirer: Any = None  # app.expiry.Expirer (timed deletions of broadcasts) or None
+    syncer: Any = None  # app.syncer.Syncer (keeps My posts in step with the channels) or None
+    sync_ignore: set = field(default_factory=set)  # (channel id, message id) of posts the owner made the bot forget
 
 
 def is_cb(event) -> bool:

app/config.py

@@ -1,6 +1,7 @@
 """Settings, read from environment variables (or a .env file)."""
 from __future__ import annotations
 
+import math
 import os
 import tempfile
 from dataclasses import dataclass
@@ -55,6 +56,12 @@ def _float(name: str, default: float) -> float:
         return default
 
 
+def _whole(name: str, default: int) -> int:
+    """A whole number from the environment; anything that is not a finite number gives the default."""
+    value = _float(name, float(default))
+    return int(value) if math.isfinite(value) else default
+
+
 @dataclass(frozen=True)
 class Config:
     api_id: int
@@ -73,6 +80,8 @@ class Config:
     error_log_chat_id: Optional[int] = None  # errors are sent to this chat; None = off
     userbot_session: str = ""  # session text of the helper account that deletes old posts
     userbot_keep_admin: bool = False  # keep the userbot an admin after a clean-up (default: take the right away)
+    delete_service_messages: bool = True  # remove "channel name changed", "pinned a message", video chat ... notices
+    sync_interval_minutes: int = 60  # how often every channel is compared with My posts (0 = never on its own)
 
     def is_owner(self, uid) -> bool:
         return uid in self.owners
@@ -115,4 +124,6 @@ def load_config() -> Config:
         error_log_chat_id=_chat_id("ERROR_LOG_CHAT_ID", DEFAULT_ERROR_LOG_CHAT_ID),
         userbot_session=(os.getenv("USERBOT_SESSION") or "").strip(),
         userbot_keep_admin=_bool("USERBOT_KEEP_ADMIN", False),
+        delete_service_messages=_bool("DELETE_SERVICE_MESSAGES", True),
+        sync_interval_minutes=max(0, _whole("SYNC_INTERVAL_MINUTES", 60)),
     )

app/db.py

@@ -21,8 +21,10 @@ from sqlalchemy import (
     desc,
     func,
     select,
+    update,
 )
 from sqlalchemy.dialects.postgresql import JSONB
+from sqlalchemy.exc import IntegrityError
 from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
 from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
 from sqlalchemy.pool import NullPool
@@ -133,6 +135,72 @@ class MigrationItem(Base):
     new_deleted: Mapped[bool] = mapped_column(Boolean, default=False)
 
 
+class Shift(Base):
+    """One /shift run: posts of a source channel copied into another channel (so it can be continued or undone)."""
+
+    __tablename__ = "shifts"
+
+    id: Mapped[str] = mapped_column(String(32), primary_key=True)
+    src_channel_id: Mapped[int] = mapped_column(BigInteger, index=True)
+    src_access_hash: Mapped[int] = mapped_column(BigInteger, default=0)
+    src_title: Mapped[str] = mapped_column(String(256), default="")
+    src_username: Mapped[Optional[str]] = mapped_column(String(64), nullable=True)
+    dst_channel_id: Mapped[int] = mapped_column(BigInteger, index=True)
+    first_id: Mapped[int] = mapped_column(BigInteger)
+    last_id: Mapped[int] = mapped_column(BigInteger)
+    # copying | stopped | incomplete | copied | copies_deleted
+    status: Mapped[str] = mapped_column(String(16), default="copying")
+    created_by: Mapped[int] = mapped_column(BigInteger, default=0)
+    created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
+
+
+class ShiftItem(Base):
+    __tablename__ = "shift_items"
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
+    shift_id: Mapped[str] = mapped_column(String(32), ForeignKey("shifts.id"), index=True)
+    old_id: Mapped[int] = mapped_column(BigInteger)  # message id in the source channel
+    new_id: Mapped[int] = mapped_column(BigInteger)  # message id of the copy in the destination channel
+    new_deleted: Mapped[bool] = mapped_column(Boolean, default=False)
+
+
+class ScheduledDelete(Base):
+    """A channel message the bot has to delete at a given time (a broadcast with an expiry)."""
+
+    __tablename__ = "scheduled_deletes"
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
+    channel_id: Mapped[int] = mapped_column(BigInteger, index=True)
+    message_id: Mapped[int] = mapped_column(BigInteger)
+    delete_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), index=True)
+    group_id: Mapped[str] = mapped_column(String(32), default="")  # the broadcast it belongs to
+    status: Mapped[str] = mapped_column(String(12), default="pending")  # pending | failed
+    attempts: Mapped[int] = mapped_column(Integer, default=0)
+    last_error: Mapped[Optional[str]] = mapped_column(String(160), nullable=True)
+    created_by: Mapped[int] = mapped_column(BigInteger, default=0)
+    created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
+
+
+class ChannelSync(Base):
+    """Where the last look at a channel stopped, so the next one only has to read what is new."""
+
+    __tablename__ = "channel_sync"
+
+    channel_id: Mapped[int] = mapped_column(BigInteger, primary_key=True, autoincrement=False)
+    last_top: Mapped[int] = mapped_column(BigInteger, default=0)  # the newest message id that existed then
+    last_pts: Mapped[int] = mapped_column(BigInteger, default=0)  # the channel's event counter then
+    deferred: Mapped[int] = mapped_column(Integer, default=0)  # new messages left for the next look (too fresh then)
+    last_run: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
+    last_deep: Mapped[Optional[datetime]] = mapped_column(DateTime(timezone=True), nullable=True)  # last look at every id
+
+
+def as_utc(dt: Optional[datetime]) -> Optional[datetime]:
+    """A datetime from the database as an aware UTC one (SQLite hands back naive values)."""
+    if dt is None:
+        return None
+    return dt.replace(tzinfo=timezone.utc) if dt.tzinfo is None else dt.astimezone(timezone.utc)
+
+
 def normalize_db_url(url: str) -> tuple:
     """Turn a plain DATABASE_URL into a SQLAlchemy async URL + connect args."""
     connect_args: dict = {}
@@ -303,6 +371,21 @@ class Database:
             )
             return list((await s.execute(q)).scalars())
 
+    async def sent_posts_from(self, channel_id: int, first_message_id: int) -> list:
+        """Published posts of a channel from message id `first_message_id` on, oldest first."""
+        async with self.Session() as s:
+            q = (
+                select(Post)
+                .where(
+                    Post.channel_id == channel_id,
+                    Post.status == "sent",
+                    Post.message_id.is_not(None),
+                    Post.message_id >= first_message_id,
+                )
+                .order_by(Post.message_id, Post.id)
+            )
+            return list((await s.execute(q)).scalars())
+
     async def mine_message_ids(self, channel_id: int) -> set:
         async with self.Session() as s:
             q = select(Post.message_id).where(
@@ -315,6 +398,162 @@ class Database:
             q = select(Post).where(Post.channel_id == channel_id, Post.message_id == message_id)
             return (await s.execute(q)).scalar_one_or_none()
 
+    async def forget_posts(self, channel_id: int, message_ids: list) -> int:
+        """Remove the saved posts of these channel messages (they were deleted in the channel)."""
+        if not message_ids:
+            return 0
+        async with self.Session() as s:
+            rows = (
+                await s.execute(select(Post).where(Post.channel_id == channel_id, Post.message_id.in_(list(message_ids))))
+            ).scalars().all()
+            for p in rows:
+                await s.delete(p)
+            await s.commit()
+            return len(rows)
+
+    async def posts_at(self, channel_id: int, message_ids: list) -> list:
+        """The published posts that sit at these message ids."""
+        if not message_ids:
+            return []
+        async with self.Session() as s:
+            q = select(Post).where(
+                Post.channel_id == channel_id, Post.status == "sent", Post.message_id.in_(list(message_ids))
+            )
+            return list((await s.execute(q)).scalars())
+
+    async def release_slot(self, channel_id: int, message_id: int, keep_pid: Optional[int] = None) -> int:
+        """Drop any other saved post at this message id (the sync may have picked the message up a moment earlier)."""
+        async with self.Session() as s:
+            q = select(Post).where(Post.channel_id == channel_id, Post.message_id == message_id)
+            if keep_pid is not None:
+                q = q.where(Post.id != keep_pid)
+            rows = list((await s.execute(q)).scalars())
+            for p in rows:
+                await s.delete(p)
+            if rows:
+                await s.commit()
+            return len(rows)
+
+    async def _drop_slots(self, s, channel_id: int, message_ids: list) -> None:
+        """Inside a transaction: remove saved posts at these ids, so the posts about to be saved can take their place."""
+        q = select(Post).where(Post.channel_id == channel_id, Post.message_id.in_(list(message_ids)))
+        rows = list((await s.execute(q)).scalars())
+        for p in rows:
+            await s.delete(p)
+        if rows:
+            await s.flush()
+
+    # ------------------------------------------------------- what happens outside the bot
+    async def get_sync(self, channel_id: int) -> Optional[ChannelSync]:
+        async with self.Session() as s:
+            return await s.get(ChannelSync, channel_id)
+
+    async def max_saved_id(self, channel_id: int) -> int:
+        """The newest message id among the channel's published posts (0 if there is none)."""
+        async with self.Session() as s:
+            q = select(func.max(Post.message_id)).where(
+                Post.channel_id == channel_id, Post.status == "sent", Post.message_id.is_not(None)
+            )
+            return int((await s.execute(q)).scalar_one_or_none() or 0)
+
+    async def save_sync(
+        self, channel_id: int, *, last_top: int, last_pts: int, deferred: int = 0, deep: bool = False
+    ) -> None:
+        async with self.Session() as s:
+            row = await s.get(ChannelSync, channel_id)
+            if row is None:
+                row = ChannelSync(channel_id=channel_id)
+                s.add(row)
+            now = utcnow()
+            row.last_top, row.last_pts, row.deferred, row.last_run = last_top, last_pts, deferred, now
+            if deep:
+                row.last_deep = now
+            await s.commit()
+
+    async def adopt_message(self, channel_id: int, message_id: int, fields: dict, sent_at: Optional[datetime] = None) -> tuple:
+        """Save a post somebody else made in the channel. Returns (post, True), or (None, False) when the bot already
+        has a post at that message id (also when another task saved it a moment ago)."""
+        async with self.Session() as s:
+            q = select(Post).where(Post.channel_id == channel_id, Post.message_id == message_id)
+            if (await s.execute(q)).scalar_one_or_none() is not None:
+                return None, False
+            p = Post(channel_id=channel_id, message_id=message_id, status="sent", source="adopted", created_by=0,
+                     sent_at=sent_at or utcnow(), **fields)
+            s.add(p)
+            try:
+                await s.commit()
+            except IntegrityError:
+                await s.rollback()
+                return None, False
+            return p, True
+
+    async def sync_update_post(self, pid: int, expected: Optional[datetime], fields: dict) -> bool:
+        """Save what the channel shows - but only if the saved post is still exactly as it was when it was read
+        (`expected` is its updated_at then), so a change made through the bot meanwhile is never overwritten."""
+        async with self.Session() as s:
+            res = await s.execute(
+                update(Post).where(Post.id == pid, Post.updated_at == expected).values(**fields, updated_at=utcnow())
+            )
+            await s.commit()
+            return bool(res.rowcount)
+
+    # ------------------------------------------------------------ timed deletions
+    async def schedule_deletes(self, rows: list) -> None:
+        """rows: dicts with channel_id, message_id, delete_at, group_id, created_by."""
+        if not rows:
+            return
+        async with self.Session() as s:
+            s.add_all([ScheduledDelete(**r) for r in rows])
+            await s.commit()
+
+    async def due_deletes(self, now: datetime, limit: int = 500) -> list:
+        async with self.Session() as s:
+            q = (
+                select(ScheduledDelete)
+                .where(ScheduledDelete.status == "pending", ScheduledDelete.delete_at <= now)
+                .order_by(ScheduledDelete.delete_at, ScheduledDelete.id)
+                .limit(limit)
+            )
+            return list((await s.execute(q)).scalars())
+
+    async def next_delete_at(self) -> Optional[datetime]:
+        async with self.Session() as s:
+            q = select(func.min(ScheduledDelete.delete_at)).where(ScheduledDelete.status == "pending")
+            return as_utc((await s.execute(q)).scalar_one_or_none())
+
+    async def pending_deletes(self, group_id: Optional[str] = None, status: str = "pending") -> list:
+        async with self.Session() as s:
+            q = select(ScheduledDelete).where(ScheduledDelete.status == status).order_by(ScheduledDelete.id)
+            if group_id:
+                q = q.where(ScheduledDelete.group_id == group_id)
+            return list((await s.execute(q)).scalars())
+
+    async def finish_deletes(self, ids: list) -> None:
+        if not ids:
+            return
+        async with self.Session() as s:
+            for row in (await s.execute(select(ScheduledDelete).where(ScheduledDelete.id.in_(list(ids))))).scalars():
+                await s.delete(row)
+            await s.commit()
+
+    async def retry_delete(self, sid: int, *, at: datetime, error: str) -> None:
+        async with self.Session() as s:
+            row = await s.get(ScheduledDelete, sid)
+            if row is not None:
+                row.attempts += 1
+                row.delete_at = at
+                row.last_error = error[:160]
+                await s.commit()
+
+    async def fail_delete(self, sid: int, error: str) -> None:
+        async with self.Session() as s:
+            row = await s.get(ScheduledDelete, sid)
+            if row is not None:
+                row.attempts += 1
+                row.status = "failed"
+                row.last_error = error[:160]
+                await s.commit()
+
     # ------------------------------------------------------------- replace history
     async def create_batch(self, batch_id: str, old: str, new: str, user_id: int) -> None:
         async with self.Session() as s:
@@ -441,12 +680,46 @@ class Database:
         if not rows:
             return
         async with self.Session() as s:
+            await self._drop_slots(s, channel_id, [r["new_id"] for r in rows])
             for r in rows:
                 s.add(MigrationItem(migration_id=mid, old_id=r["old_id"], new_id=r["new_id"]))
                 s.add(Post(channel_id=channel_id, message_id=r["new_id"], status="sent", source="bot",
                            created_by=user_id, sent_at=utcnow(), **r["post"]))
             await s.commit()
 
+    async def migration_pairs(
+        self, mid: Optional[str] = None, channel_id: Optional[int] = None, alive_only: bool = True,
+        only_deleted_old: bool = False,
+    ) -> list:
+        """[(old id, new id)] of the copies of one repost (`mid`) or of every repost of a channel, oldest repost first.
+        alive_only: leave out copies that were removed again (undo). only_deleted_old: only posts whose original
+        has been deleted (links to those are dead)."""
+        async with self.Session() as s:
+            q = select(MigrationItem.old_id, MigrationItem.new_id).join(Migration, Migration.id == MigrationItem.migration_id)
+            if mid is not None:
+                q = q.where(MigrationItem.migration_id == mid)
+            if channel_id is not None:
+                q = q.where(Migration.channel_id == channel_id)
+            if alive_only:
+                q = q.where(MigrationItem.new_deleted.is_(False))
+            if only_deleted_old:
+                q = q.where(MigrationItem.old_deleted.is_(True))
+            q = q.order_by(Migration.created_at, MigrationItem.id)
+            return [(a, b) for a, b in (await s.execute(q)).all()]
+
+    async def update_post_by_message(self, channel_id: int, message_id: int, **fields: Any) -> Optional[Post]:
+        """Change the saved copy of the post that sits at `message_id` (None if the bot does not know that post)."""
+        async with self.Session() as s:
+            q = select(Post).where(Post.channel_id == channel_id, Post.message_id == message_id)
+            p = (await s.execute(q)).scalar_one_or_none()
+            if p is None:
+                return None
+            for k, v in fields.items():
+                setattr(p, k, v)
+            p.updated_at = utcnow()
+            await s.commit()
+            return p
+
     async def migration_pending_ids(self, mid: str, which: str, limit: int = 100) -> list:
         col, flag = (MigrationItem.old_id, MigrationItem.old_deleted) if which == "old" else (MigrationItem.new_id, MigrationItem.new_deleted)
         async with self.Session() as s:
@@ -478,6 +751,101 @@ class Database:
             hi = (await s.execute(select(func.max(MigrationItem.new_id)).where(MigrationItem.migration_id == mid))).scalar_one()
             return {"copied": total, "old_left": old_left, "new_left": new_left, "first_new": lo, "last_new": hi}
 
+    # ------------------------------------------------------------------- /shift
+    SHIFT_OPEN = ("copying", "stopped", "incomplete")
+
+    async def create_shift(self, sid: str, *, src, dst_channel_id: int, first_id: int, last_id: int, user_id: int) -> None:
+        """`src`: any object with id, access_hash, title and username (the source channel need not be registered)."""
+        async with self.Session() as s:
+            s.add(Shift(id=sid, src_channel_id=src.id, src_access_hash=src.access_hash or 0, src_title=src.title or "",
+                        src_username=src.username, dst_channel_id=dst_channel_id, first_id=first_id, last_id=last_id,
+                        created_by=user_id))
+            await s.commit()
+
+    async def get_shift(self, sid: str) -> Optional[Shift]:
+        async with self.Session() as s:
+            return await s.get(Shift, sid)
+
+    async def open_shift(self, src_id: Optional[int] = None, dst_id: Optional[int] = None) -> Optional[Shift]:
+        """The newest shift that is not finished (still copying, stopped, or incomplete)."""
+        async with self.Session() as s:
+            q = select(Shift).where(Shift.status.in_(self.SHIFT_OPEN))
+            if src_id is not None:
+                q = q.where(Shift.src_channel_id == src_id)
+            if dst_id is not None:
+                q = q.where(Shift.dst_channel_id == dst_id)
+            return (await s.execute(q.order_by(desc(Shift.created_at)).limit(1))).scalar_one_or_none()
+
+    async def set_shift_status(self, sid: str, status: str) -> None:
+        async with self.Session() as s:
+            sh = await s.get(Shift, sid)
+            if sh is not None:
+                sh.status = status
+                await s.commit()
+
+    async def shift_done_ids(self, sid: str) -> set:
+        async with self.Session() as s:
+            q = select(ShiftItem.old_id).where(ShiftItem.shift_id == sid)
+            return set((await s.execute(q)).scalars())
+
+    async def shift_pairs(self, sid: str, alive_only: bool = True) -> list:
+        """[(source id, id of the copy)] of one shift, in copy order."""
+        async with self.Session() as s:
+            q = select(ShiftItem.old_id, ShiftItem.new_id).where(ShiftItem.shift_id == sid)
+            if alive_only:
+                q = q.where(ShiftItem.new_deleted.is_(False))
+            return [(a, b) for a, b in (await s.execute(q.order_by(ShiftItem.id))).all()]
+
+    async def shifted_ids(self, src_id: int, dst_id: int, first: int, last: int) -> set:
+        """Source ids in first..last that already have a live copy in `dst_id` from an earlier shift."""
+        async with self.Session() as s:
+            q = (
+                select(ShiftItem.old_id)
+                .join(Shift, Shift.id == ShiftItem.shift_id)
+                .where(Shift.src_channel_id == src_id, Shift.dst_channel_id == dst_id, ShiftItem.new_deleted.is_(False),
+                       ShiftItem.old_id >= first, ShiftItem.old_id <= last)
+            )
+            return set((await s.execute(q)).scalars())
+
+    async def record_shift(self, sid: str, dst_channel_id: int, rows: list, user_id: int) -> None:
+        """rows: [{old_id, new_id, post: {Post fields}}] - saves the mapping and registers the copies in My posts."""
+        if not rows:
+            return
+        async with self.Session() as s:
+            await self._drop_slots(s, dst_channel_id, [r["new_id"] for r in rows])
+            for r in rows:
+                s.add(ShiftItem(shift_id=sid, old_id=r["old_id"], new_id=r["new_id"]))
+                s.add(Post(channel_id=dst_channel_id, message_id=r["new_id"], status="sent", source="bot",
+                           created_by=user_id, sent_at=utcnow(), **r["post"]))
+            await s.commit()
+
+    async def shift_counts(self, sid: str) -> dict:
+        async with self.Session() as s:
+            base = ShiftItem.shift_id == sid
+            total = (await s.execute(select(func.count()).select_from(ShiftItem).where(base))).scalar_one()
+            left = (await s.execute(select(func.count()).select_from(ShiftItem).where(base, ShiftItem.new_deleted.is_(False)))).scalar_one()
+            lo = (await s.execute(select(func.min(ShiftItem.new_id)).where(base))).scalar_one()
+            hi = (await s.execute(select(func.max(ShiftItem.new_id)).where(base))).scalar_one()
+            return {"copied": total, "new_left": left, "first_new": lo, "last_new": hi}
+
+    async def shift_pending_new(self, sid: str, limit: int = 100) -> list:
+        async with self.Session() as s:
+            q = select(ShiftItem.new_id).where(ShiftItem.shift_id == sid, ShiftItem.new_deleted.is_(False)).order_by(ShiftItem.new_id).limit(limit)
+            return list((await s.execute(q)).scalars())
+
+    async def mark_shift_deleted(self, sid: str, dst_channel_id: int, ids: list) -> None:
+        """The copies are gone from Telegram: flag them and drop their saved posts."""
+        if not ids:
+            return
+        async with self.Session() as s:
+            q = select(ShiftItem).where(ShiftItem.shift_id == sid, ShiftItem.new_id.in_(ids))
+            for it in (await s.execute(q)).scalars():
+                it.new_deleted = True
+            pq = select(Post).where(Post.channel_id == dst_channel_id, Post.message_id.in_(ids))
+            for p in (await s.execute(pq)).scalars():
+                await s.delete(p)
+            await s.commit()
+
     # -------------------------------------------------------------------- export
     async def export_all(self) -> dict:
         def clean(d: dict) -> dict:

app/expiry.py

@@ -0,0 +1,208 @@
+"""Timed deletion: a broadcast with an expiry ("/broadcast 1h") is removed from every channel - and from My posts -
+when its time is up.
+
+The deletions are saved in the database (table scheduled_deletes), so a restart loses nothing: posts that came due while
+the bot was off are deleted as soon as it is back. The Expirer wakes up only when something is due; it does not poll the
+database, so a sleeping serverless database stays asleep.
+"""
+from __future__ import annotations
+
+import asyncio
+import logging
+import re
+from datetime import datetime, timedelta
+from typing import Optional
+
+from .db import as_utc, utcnow
+from .repost_engine import delete_batch
+from .tgutil import esc, peer_of
+
+log = logging.getLogger(__name__)
+
+MIN_SECONDS = 60
+MAX_SECONDS = 366 * 86400
+MAX_ATTEMPTS = 5
+RETRY_AFTER = 600  # seconds between attempts, times the attempt number
+LOCK_WAIT = 60  # a long job (repost...) is running: look again in a minute
+
+_UNITS = {
+    "m": 60, "min": 60, "mins": 60, "minute": 60, "minutes": 60,
+    "h": 3600, "hr": 3600, "hrs": 3600, "hour": 3600, "hours": 3600,
+    "d": 86400, "day": 86400, "days": 86400,
+    "w": 604800, "week": 604800, "weeks": 604800,
+}  # fmt: skip
+_PART = re.compile(r"(\d+)\s*([a-z]+)")
+DURATION_HELP = "Use minutes, hours or days: 50m, 1h, 5d (or 1d12h). Leave the time out to keep the message forever."
+
+
+def parse_duration(text: str) -> int:
+    """'50m' -> 3000, '1h' -> 3600, '5d' -> 432000, '1d12h' -> 129600 (seconds). ValueError with a message if unusable."""
+    t = (text or "").strip().lower().replace(",", " ")
+    if not t or not re.fullmatch(r"(?:\d+\s*[a-z]+\s*)+", t):
+        raise ValueError(f"I can't read “{(text or '').strip()[:30]}” as a time. {DURATION_HELP}")
+    total = 0
+    for n, unit in _PART.findall(t):
+        if unit not in _UNITS:
+            raise ValueError(f"“{n}{unit}” - I don't know the unit “{unit}”. {DURATION_HELP}")
+        total += int(n) * _UNITS[unit]
+    if total < MIN_SECONDS:
+        raise ValueError("The shortest time is 1 minute.")
+    if total > MAX_SECONDS:
+        raise ValueError("The longest time is 1 year (365d).")
+    return total
+
+
+def human(seconds: int) -> str:
+    """3000 -> '50 minutes', 3600 -> '1 hour', 129600 -> '1 day 12 hours'."""
+    seconds = int(seconds)
+    days, rest = divmod(seconds, 86400)
+    hours, rest = divmod(rest, 3600)
+    minutes = rest // 60
+    parts = []
+    for n, word in ((days, "day"), (hours, "hour"), (minutes, "minute")):
+        if n:
+            parts.append(f"{n} {word}" + ("s" if n != 1 else ""))
+    return " ".join(parts[:2]) or "under a minute"
+
+
+def stamp(dt: datetime) -> str:
+    return as_utc(dt).strftime("%d %b %Y, %H:%M UTC")
+
+
+class Expirer:
+    """Deletes scheduled messages when they come due. `start()` runs it in the background."""
+
+    def __init__(self, ctx, *, notify=None):
+        self.ctx = ctx
+        self._next: Optional[datetime] = None
+        self._wake = asyncio.Event()
+        self._task: Optional[asyncio.Task] = None
+        self._notify = notify  # async fn(user_id, text) used to tell someone a deletion failed for good
+
+    # ------------------------------------------------------------------------- running
+    def start(self) -> None:
+        if self._task is None:
+            self._task = asyncio.create_task(self.run(), name="expirer")
+
+    async def stop(self) -> None:
+        if self._task is not None:
+            self._task.cancel()
+            try:
+                await self._task
+            except (asyncio.CancelledError, Exception):
+                pass
+            self._task = None
+
+    def notify(self, when: datetime) -> None:
+        """Something was scheduled for `when`: make sure the loop wakes up in time."""
+        when = as_utc(when)
+        if self._next is None or when < self._next:
+            self._next = when
+        self._wake.set()
+
+    async def load(self) -> None:
+        """The one look at the database (when the bot starts): when is the first deletion due?"""
+        try:
+            self._next = await self.ctx.db.next_delete_at()
+        except Exception:
+            log.exception("could not read the scheduled deletions")
+
+    async def run(self) -> None:
+        await self.load()
+        while True:
+            delay = None if self._next is None else max(0.0, (self._next - utcnow()).total_seconds())
+            try:
+                await asyncio.wait_for(self._wake.wait(), timeout=delay)
+            except asyncio.TimeoutError:
+                pass
+            self._wake.clear()
+            if self._next is not None and self._next <= utcnow():
+                try:
+                    await self.sweep()
+                except asyncio.CancelledError:
+                    raise
+                except Exception:
+                    log.exception("deleting scheduled messages failed")
+                    self._next = utcnow() + timedelta(seconds=RETRY_AFTER)
+
+    # --------------------------------------------------------------------------- work
+    async def sweep(self, now: Optional[datetime] = None) -> dict:
+        """Delete everything that is due at `now`. Returns {'deleted': n, 'retry': n, 'failed': n}."""
+        ctx, db = self.ctx, self.ctx.db
+        now = now or utcnow()
+        stats = {"deleted": 0, "retry": 0, "failed": 0, "waiting": False}
+        due = await db.due_deletes(now)
+        if due and ctx.lock.locked():  # a repost / shift is running: don't pull posts out from under it
+            stats["waiting"] = True
+            self._next = now + timedelta(seconds=LOCK_WAIT)
+            return stats
+        by_channel: dict = {}
+        for row in due:
+            by_channel.setdefault(row.channel_id, []).append(row)
+        for cid, rows in by_channel.items():
+            await self._sweep_channel(cid, rows, now, stats)
+        self._next = await db.next_delete_at()
+        return stats
+
+    async def _sweep_channel(self, cid: int, rows: list, now: datetime, stats: dict) -> None:
+        ctx, db = self.ctx, self.ctx.db
+        ch = await db.get_channel(cid)
+        if ch is None:
+            for r in rows:
+                await db.fail_delete(r.id, "the channel was removed from the bot")
+                stats["failed"] += 1
+            return
+        peer = peer_of(ch)
+        ids = sorted({r.message_id for r in rows})
+        gone: set = set()
+        error: Optional[str] = None
+        try:
+            for i in range(0, len(ids), 100):
+                out = await delete_batch(ctx.client, peer, ids[i : i + 100], None)
+                gone |= out.gone
+                error = error or out.bot_error
+            left = [m for m in ids if m not in gone]
+            ub = getattr(ctx, "userbot", None)
+            if left and ub is not None and ub.enabled:  # the bot may not delete old posts: the userbot tries
+                fallback, deleter = await ub.fallback_for(ch)
+                try:
+                    if fallback is not None:
+                        for i in range(0, len(left), 100):
+                            out = await delete_batch(ctx.client, peer, left[i : i + 100], fallback)
+                            gone |= out.gone
+                            error = out.fallback_error or error
+                finally:
+                    if deleter is not None:
+                        await deleter.close()
+        except Exception as e:
+            log.exception("deleting scheduled messages of channel %s failed", cid)
+            error = type(e).__name__
+        done = [r for r in rows if r.message_id in gone]
+        await db.finish_deletes([r.id for r in done])
+        await db.forget_posts(cid, sorted(gone))
+        stats["deleted"] += len(done)
+        for r in rows:
+            if r.message_id in gone:
+                continue
+            why = error or "Telegram did not delete it"
+            if r.attempts + 1 >= MAX_ATTEMPTS:
+                await db.fail_delete(r.id, why)
+                stats["failed"] += 1
+                await self._tell(r, ch, why)
+            else:
+                await db.retry_delete(r.id, at=now + timedelta(seconds=RETRY_AFTER * (r.attempts + 1)), error=why)
+                stats["retry"] += 1
+
+    async def _tell(self, row, ch, why: str) -> None:
+        """A deletion failed for good: tell the person who sent the broadcast."""
+        text = (
+            f"⚠️ I could not delete the expired broadcast post {row.message_id} in <b>{esc(ch.title)}</b> "
+            f"({esc(why)}). Delete it by hand, then use “Only forget it in the bot” in /posts."
+        )
+        try:
+            if self._notify is not None:
+                await self._notify(row.created_by, text)
+            elif row.created_by:
+                await self.ctx.client.send_message(row.created_by, text, link_preview=False)
+        except Exception:
+            log.warning("could not tell user %s about the failed deletion", row.created_by, exc_info=True)

app/handlers/autopost.py

@@ -0,0 +1,184 @@
+"""/autopost <total> <interval>: post "Episodes 01 to 20", "Episodes 21 to 40", ... with a download button each,
+then ask for the real link of every button. Owners and admins."""
+from __future__ import annotations
+
+import asyncio
+import logging
+import re
+import secrets
+import time
+from dataclasses import dataclass, field
+from types import SimpleNamespace
+
+from telethon import Button, errors
+
+from ..autopost import (
+    MAX_POSTS,
+    PLACEHOLDER_LINK,
+    USAGE,
+    button_label,
+    parse_autopost_args,
+    plan_ranges,
+    post_buttons,
+    post_text,
+    ranges_summary,
+    title_of,
+)
+from ..common import (
+    Ctx,
+    UserError,
+    clear_state,
+    cmd,
+    discard_temp,
+    edit_callback_message,
+    guard,
+    on_cb,
+    purge_pending,
+    say,
+    send_post,
+)
+from ..db import utcnow
+from ..tgutil import esc, explain_rpc, flood_retry, get_rights, peer_of, short
+from .masslinks import begin_queue
+
+log = logging.getLogger(__name__)
+
+
+@dataclass
+class AutoChoice:
+    """/autopost was typed with good numbers; the channel still has to be picked."""
+
+    user: int
+    total: int
+    interval: int
+    ranges: list
+    created: float = field(default_factory=time.monotonic)
+
+
+@dataclass
+class AutoResult:
+    posts: list = field(default_factory=list)  # the saved Post rows, in the order they were posted
+    failure: str = ""  # why it stopped before the end (empty = all posted)
+
+
+async def post_series(ctx: Ctx, ch, ranges: list, user_id: int, *, delay: float = 1.0, progress=None) -> AutoResult:
+    """Send one post per range to `ch` and save each in My posts. Stops at the first post Telegram refuses."""
+    res = AutoResult()
+    for n, (a, b) in enumerate(ranges, 1):
+        text, buttons = post_text(a, b), post_buttons(a, b)
+        shape = SimpleNamespace(text=text, entities=[], buttons=buttons, media_file_id=None, link_preview=False)
+        try:
+            sent = await flood_retry(lambda: send_post(ctx.client, peer_of(ch), shape))
+        except errors.RPCError as e:
+            res.failure = f"{title_of(a, b)}: {explain_rpc(e)}"
+            break
+        except Exception as e:
+            log.exception("autopost: posting %s failed", title_of(a, b))
+            res.failure = f"{title_of(a, b)}: {type(e).__name__}"
+            break
+        res.posts.append(
+            await ctx.db.create_post(
+                channel_id=ch.id,
+                message_id=sent.id,
+                status="sent",
+                source="bot",
+                created_by=user_id,
+                text=text,
+                entities=[],
+                buttons=buttons,
+                link_preview=False,
+                sent_at=utcnow(),
+            )
+        )
+        if progress:
+            await progress(n)
+        if n < len(ranges):
+            await asyncio.sleep(delay)
+    return res
+
+
+def register(ctx: Ctx) -> None:
+    client, db, cfg = ctx.client, ctx.db, ctx.cfg
+
+    @client.on(cmd("autopost", args=True))
+    @guard(ctx)
+    async def h_autopost(event):
+        purge_pending(ctx)
+        raw = (event.pattern_match.group(1) or "").strip()
+        if not raw:
+            await say(event, USAGE)
+            return
+        try:
+            total, interval = parse_autopost_args(raw)
+        except ValueError as e:
+            raise UserError(f"{e}\n\n" + re.sub(r"<[^>]+>", "", USAGE).replace("&lt;", "<").replace("&gt;", ">"))
+        ranges = plan_ranges(total, interval)
+        if len(ranges) > MAX_POSTS:
+            raise UserError(f"That would be {len(ranges)} posts - at most {MAX_POSTS} at once. Use a bigger interval.")
+        chans = await db.list_channels()
+        if not chans:
+            raise UserError("No channels yet. Register one with /addchannel first.")
+        uid = event.sender_id
+        await discard_temp(ctx, uid)
+        clear_state(ctx, uid)
+        token = secrets.token_hex(4)
+        ctx.pending[token] = AutoChoice(user=uid, total=total, interval=interval, ranges=ranges)
+        first = ranges[0]
+        kb = [[Button.inline(f"📢 {short(c.title, 40)}", f"apc:{token}:{c.id}")] for c in chans]
+        kb.append([Button.inline("✖️ Cancel", f"apx:{token}")])
+        await say(
+            event,
+            f"📋 <b>Autopost</b>: {len(ranges)} post(s) for {total} episodes (every {interval})\n"
+            f"{ranges_summary(ranges)}\n\n"
+            f"Each post says “{esc(title_of(*first))}” with the button “{esc(button_label(*first))}”. The buttons get "
+            f"the link {esc(PLACEHOLDER_LINK)} for now; when everything is posted I ask you for the real link of "
+            "each button, one after the other.\n\n"
+            "📢 <b>Which channel should I post them in?</b>",
+            kb,
+        )
+
+    @on_cb(ctx, "apc")
+    async def cb_pick(event, parts):
+        token, cid = parts[0], int(parts[1])
+        choice = ctx.pending.get(token)
+        if not isinstance(choice, AutoChoice) or choice.user != event.sender_id:
+            raise UserError("That list has expired. Run /autopost again.")
+        ch = await db.get_channel(cid)
+        if ch is None or not ch.active:
+            raise UserError("That channel is not registered any more. Run /autopost again.")
+        rights = await get_rights(client, ch)
+        if rights is not None and not (rights.admin and rights.post):
+            raise UserError("The bot needs the “Post messages” admin right in that channel.")
+        if ctx.lock.locked():
+            raise UserError("Another long job (replace / repost / shift) is still running. Try again in a minute.")
+        ctx.pending.pop(token, None)  # a second press on the list can't post everything twice
+        last = [0.0]
+
+        async def prog(n):
+            if time.monotonic() - last[0] < 3:
+                return
+            last[0] = time.monotonic()
+            try:
+                await edit_callback_message(event, f"📤 Posting in <b>{esc(ch.title)}</b>: {n} of {len(choice.ranges)}…")
+            except Exception:
+                pass
+
+        async with ctx.lock:
+            res = await post_series(ctx, ch, choice.ranges, event.sender_id, delay=cfg.edit_delay, progress=prog)
+        if not res.posts:
+            raise UserError(f"Nothing was posted. {res.failure}")
+        done = [(p.id, 0, 0) for p in res.posts]
+        lines = [
+            f"✅ Posted {len(res.posts)} of {len(choice.ranges)} in <b>{esc(ch.title)}</b>",
+            ranges_summary(choice.ranges[: len(res.posts)]),
+            f"Their buttons open {esc(PLACEHOLDER_LINK)} until you send the real links.",
+        ]
+        if res.failure:
+            lines.append(f"\n⚠️ Stopped early - {esc(res.failure)}\nThe rest was not posted; run /autopost again for it.")
+        await edit_callback_message(event, "\n".join(lines))
+        await begin_queue(ctx, event, kind="auto", cid=ch.id, items=done, new=True)
+
+    @on_cb(ctx, "apx")
+    async def cb_cancel(event, parts):
+        ctx.pending.pop(parts[0], None)
+        await edit_callback_message(event, "Cancelled - nothing was posted.")

app/handlers/basic.py

@@ -1,7 +1,6 @@
-"""/start /help /cancel /export /userbot /testerror, plus the routers for free text and inline-button presses."""
+"""/start /help /cancel /export /userbot, plus the routers for free text and inline-button presses."""
 from __future__ import annotations
 
-import asyncio
 import io
 import json
 import logging
@@ -17,17 +16,23 @@ HELP = (
     "<b>Channel Post Bot</b>\n\n"
     "<b>Posts</b>\n"
     "/new - create a post (text or media, with buttons)\n"
-    "/posts - your posts and drafts: edit text, caption, media and buttons, or delete them\n"
+    "/posts - your posts and drafts: edit text, caption, media and buttons, or delete them (a button post also has “Mass replace links”)\n"
+    "/autopost 114 20 - post “Episodes 01 to 20”, “21 to 40” ... up to 114, each with a download button, then fill in the links\n"
     "/cancel - stop what you're doing\n\n"
     "<b>Channels</b>\n"
     "/addchannel - register a channel (the bot must be an admin there)\n"
     "/channels - list or remove channels\n\n"
     "<b>Owner tools</b>\n"
+    "/broadcast [time] - send one message to all channels; with a time (50m, 1h, 5d) it is deleted again from all channels and My posts afterwards\n"
     "/replace @old @new - swap the username inside t.me links (button links and hyperlinks) in all channels\n"
-    "/undo - undo the last /replace\n"
-    "/repost @old @new - copy every post of a channel in order (links swapped), then delete the old posts\n"
+    "/handleswap @old @new - change the username @old written in posts (text, captions, button names) to @new in all channels; links stay\n"
+    "/undo - undo the last /replace or /handleswap\n"
+    "/repost @old @new - copy every post of a channel in order (links swapped, replies, pins and links between posts kept), then delete the old posts\n"
+    "/fixlinks - point links at the reposted copies for a channel you reposted before\n"
+    "/shift @source @destination [from_id] [to_id] - copy posts from one channel into another (buttons kept, listed in My posts)\n"
+    "/sync - compare every channel with My posts right now (posts deleted, edited or added by others, leftover service messages); "
+    "/sync --clean also deletes the old service messages that are already in the channels' history\n"
     "/userbot - status of the helper account that deletes posts the bot is not allowed to delete\n"
-    "/testerror - send a test error to the error log chat\n"
     "/export - download a JSON backup of your posts"
 )
 
@@ -78,27 +83,6 @@ def register(ctx: Ctx) -> None:
         kept = "stays an admin" if ub.keep_admin else "is made admin only while deleting"
         await say(event, f"🤖 <b>Userbot:</b> {esc(ub.describe())}\nIt {kept} (USERBOT_KEEP_ADMIN).")
 
-    @client.on(cmd("testerror", args=True))
-    @guard(ctx, owner=True)
-    async def h_testerror(event):
-        rep = getattr(ctx, "reporter", None)
-        if rep is None or not rep.enabled:
-            await say(event, "The error log is switched off (ERROR_LOG_CHAT_ID is 0 / off).")
-            return
-        mode = (event.pattern_match.group(1) or "").strip().lower()
-        if mode == "task":
-            async def boom():
-                raise RuntimeError("Test error from /testerror task")
-
-            asyncio.get_running_loop().create_task(boom())  # nobody awaits it, like a forgotten background task
-            await say(event, "A background task will fail in a moment - look in the error log chat.")
-            return
-        await say(event, "Raising a test error now - look in the error log chat. (Use <code>/testerror task</code> to test a background task.)")
-        try:
-            raise KeyError("test")
-        except KeyError as e:
-            raise RuntimeError("Test error from /testerror") from e
-
     @client.on(
         events.NewMessage(incoming=True, func=lambda e: e.is_private and not (e.raw_text or "").startswith("/"))
     )

app/handlers/broadcast.py

@@ -0,0 +1,260 @@
+"""/broadcast [time] (also spelled /boardcast): one message to every connected channel. Owner only.
+
+With a time (50m, 1h, 5d) the message is deleted again from all the channels - and from My posts - when the time is up;
+without one it stays.
+"""
+from __future__ import annotations
+
+import asyncio
+import logging
+import secrets
+import time
+from dataclasses import dataclass, field
+from datetime import datetime, timedelta
+from types import SimpleNamespace
+from typing import Optional
+
+from telethon import Button, errors
+
+from ..buttons import parse_buttons_text
+from ..common import (
+    Ctx,
+    UserError,
+    clear_state,
+    cmd,
+    discard_temp,
+    edit_callback_message,
+    guard,
+    on_cb,
+    on_text,
+    purge_pending,
+    require_owner,
+    say,
+    send_post,
+    set_state,
+    show,
+)
+from ..db import utcnow
+from ..expiry import human, parse_duration, stamp
+from ..tgutil import esc, explain_rpc, flood_retry, peer_of
+from ..ui import CANCEL
+from .panel import extract_content
+
+log = logging.getLogger(__name__)
+
+BUTTON_HELP = (
+    "➕ <b>Buttons?</b> Send one row per line as <code>Button text - link</code> "
+    "(several in one row: separate them with <code> | </code>), or skip."
+)
+
+
+@dataclass
+class BroadcastDraft:
+    """The message is made and previewed; the owner still has to confirm."""
+
+    user: int
+    content: dict  # text, entities, media_kind, media_file_id
+    buttons: list
+    ttl: Optional[int]  # seconds until it is deleted again; None = it stays
+    created: float = field(default_factory=time.monotonic)
+
+
+@dataclass
+class BroadcastResult:
+    group_id: str
+    delete_at: Optional[datetime]
+    ttl: Optional[int] = None  # seconds it stays; None = for good
+    sent: list = field(default_factory=list)  # [(channel, message id)]
+    failed: list = field(default_factory=list)  # [(channel, reason)]
+
+
+def shape_of(draft: BroadcastDraft) -> SimpleNamespace:
+    c = draft.content
+    return SimpleNamespace(
+        text=c.get("text") or "",
+        entities=c.get("entities") or [],
+        buttons=draft.buttons,
+        media_kind=c.get("media_kind"),
+        media_file_id=c.get("media_file_id"),
+        link_preview=False,
+    )
+
+
+async def send_broadcast(ctx: Ctx, chans: list, draft: BroadcastDraft, *, delay: float = 1.0, progress=None) -> BroadcastResult:
+    """Post the message in every channel, save each copy in My posts and, if it expires, schedule its deletion."""
+    db = ctx.db
+    delete_at = utcnow() + timedelta(seconds=draft.ttl) if draft.ttl else None
+    res = BroadcastResult(group_id=secrets.token_hex(6), delete_at=delete_at, ttl=draft.ttl)
+    shape = shape_of(draft)
+    for n, ch in enumerate(chans, 1):
+        try:
+            sent = await flood_retry(lambda: send_post(ctx.client, peer_of(ch), shape))
+        except errors.RPCError as e:
+            res.failed.append((ch, explain_rpc(e)))
+            continue
+        except Exception as e:
+            log.exception("broadcast: posting in channel %s failed", ch.id)
+            res.failed.append((ch, type(e).__name__))
+            continue
+        await db.create_post(
+            channel_id=ch.id,
+            message_id=sent.id,
+            status="sent",
+            source="bot",
+            created_by=draft.user,
+            text=shape.text,
+            entities=shape.entities,
+            media_kind=shape.media_kind,
+            media_file_id=shape.media_file_id,
+            buttons=shape.buttons,
+            link_preview=False,
+            sent_at=utcnow(),
+        )
+        res.sent.append((ch, sent.id))
+        if delete_at is not None:  # saved at once, so a restart in the middle loses nothing
+            await db.schedule_deletes(
+                [dict(channel_id=ch.id, message_id=sent.id, delete_at=delete_at, group_id=res.group_id, created_by=draft.user)]
+            )
+            if ctx.expirer is not None:
+                ctx.expirer.notify(delete_at)
+        if progress:
+            await progress(n)
+        if n < len(chans):
+            await asyncio.sleep(delay)
+    return res
+
+
+def report(res: BroadcastResult, total: int) -> str:
+    lines = [f"📣 <b>Broadcast sent</b> to {len(res.sent)} of {total} channel(s)"]
+    lines += [f"✅ {esc(ch.title)}" for ch, _ in res.sent]
+    lines += [f"❌ {esc(ch.title)} - {esc(why)}" for ch, why in res.failed]
+    if res.delete_at is not None and res.sent:
+        lines += ["", f"🕒 It is deleted from these channels and from My posts in {human(res.ttl)} ({stamp(res.delete_at)})."]
+    elif res.sent:
+        lines += ["", "📌 No time was given, so it stays in the channels."]
+    return "\n".join(lines)
+
+
+def register(ctx: Ctx) -> None:
+    client, db, cfg = ctx.client, ctx.db, ctx.cfg
+
+    @client.on(cmd("broadcast", args=True))
+    @client.on(cmd("boardcast", args=True))
+    @guard(ctx, owner=True)
+    async def h_broadcast(event):
+        purge_pending(ctx)
+        raw = (event.pattern_match.group(1) or "").strip()
+        ttl = None
+        if raw:
+            try:
+                ttl = parse_duration(raw)
+            except ValueError as e:
+                raise UserError(str(e))
+        chans = await db.list_channels()
+        if not chans:
+            raise UserError("No channels yet. Register one with /addchannel first.")
+        uid = event.sender_id
+        await discard_temp(ctx, uid)
+        set_state(ctx, uid, "bc_content", ttl=ttl)
+        when = (
+            f"⏳ It is deleted from all channels - and from My posts - {human(ttl)} after it is sent."
+            if ttl
+            else "📌 No time given: it stays in the channels for good. (For a timed one use e.g. <code>/broadcast 1h</code>.)"
+        )
+        await say(
+            event,
+            f"📣 <b>Broadcast</b> to {len(chans)} channel(s)\n{when}\n\n"
+            "✍️ <b>Send the message now.</b> Text, or a photo / video / GIF / file with a caption. Formatting is kept.",
+            CANCEL,
+        )
+
+    @on_text(ctx, "bc_content")
+    async def h_content(event, st):
+        m = event.message
+        if m.grouped_id and st.get("group") == m.grouped_id:
+            return  # the other items of an album: only the first one is used
+        if m.grouped_id:
+            st["group"] = m.grouped_id
+        content = extract_content(m)
+        set_state(ctx, event.sender_id, "bc_buttons", ttl=st["ttl"], content=content)
+        await say(event, BUTTON_HELP, [[Button.inline("⏭ No buttons", "brs")], CANCEL[0]])
+
+    async def confirm_step(event, st, rows: list) -> None:
+        uid = event.sender_id
+        draft = BroadcastDraft(user=uid, content=st["content"], buttons=rows, ttl=st["ttl"])
+        chans = await db.list_channels()
+        if not chans:
+            clear_state(ctx, uid)
+            raise UserError("No channels any more. Register one with /addchannel first.")
+        try:
+            await send_post(client, await event.get_input_chat(), shape_of(draft))  # the preview
+        except errors.RPCError as e:
+            raise UserError("❌ " + explain_rpc(e) + "\nThe message can't be sent like this - send it again.")
+        token = secrets.token_hex(4)
+        ctx.pending[token] = draft
+        clear_state(ctx, uid)
+        names = "\n".join(f"• {esc(c.title)}" for c in chans[:15]) + (f"\n• … and {len(chans) - 15} more" if len(chans) > 15 else "")
+        when = (
+            f"⏳ Deleted again {human(draft.ttl)} after sending."
+            if draft.ttl
+            else "📌 Permanent (no time given)."
+        )
+        await say(
+            event,
+            f"👆 That is the message. Send it to <b>{len(chans)}</b> channel(s)?\n{names}\n\n{when}",
+            [[Button.inline("📣 Send to all channels", f"brc:{token}")], [Button.inline("✖️ Cancel", f"brn:{token}")]],
+        )
+
+    @on_text(ctx, "bc_buttons")
+    async def h_buttons(event, st):
+        if event.message.grouped_id:
+            return
+        try:
+            rows = parse_buttons_text(event.raw_text or "")
+        except ValueError as e:
+            raise UserError(f"❌ {e}\n\nTry again, or tap “No buttons”.")
+        if not rows:
+            raise UserError("Send at least one button as “Text - link”, or tap “No buttons”.")
+        await confirm_step(event, st, rows)
+
+    @on_cb(ctx, "brs")
+    async def cb_no_buttons(event, parts):
+        require_owner(ctx, event)
+        st = ctx.state.get(event.sender_id)
+        if not st or st.get("mode") != "bc_buttons":
+            raise UserError("That step is no longer active. Start again with /broadcast.")
+        await confirm_step(event, st, [])
+
+    @on_cb(ctx, "brn")
+    async def cb_cancel(event, parts):
+        require_owner(ctx, event)
+        ctx.pending.pop(parts[0], None)
+        await show(event, "Cancelled - nothing was sent.")
+
+    @on_cb(ctx, "brc")
+    async def cb_send(event, parts):
+        require_owner(ctx, event)
+        token = parts[0]
+        draft = ctx.pending.get(token)
+        if not isinstance(draft, BroadcastDraft) or draft.user != event.sender_id:
+            raise UserError("That broadcast has expired. Start again with /broadcast.")
+        chans = await db.list_channels()
+        if not chans:
+            raise UserError("No channels any more. Register one with /addchannel first.")
+        if ctx.lock.locked():
+            raise UserError("Another long job (replace / repost / shift) is still running. Try again in a minute.")
+        ctx.pending.pop(token, None)  # a second press can't send it twice
+        last = [0.0]
+
+        async def prog(n):
+            if time.monotonic() - last[0] < 3:
+                return
+            last[0] = time.monotonic()
+            try:
+                await edit_callback_message(event, f"📣 Sending… {n} of {len(chans)} channel(s)")
+            except Exception:
+                pass
+
+        async with ctx.lock:
+            res = await send_broadcast(ctx, chans, draft, delay=cfg.edit_delay, progress=prog)
+        await edit_callback_message(event, report(res, len(chans)))

app/handlers/channels.py

@@ -6,6 +6,7 @@ import re
 
 from telethon import Button, events, types
 
+from ..channelref import parse_channel_ref
 from ..common import Ctx, UserError, clear_state, cmd, guard, on_cb, on_text, require_owner, say, set_state, show
 from ..tgutil import esc, get_rights, short
 from ..ui import CANCEL
@@ -25,23 +26,6 @@ NEEDS_ACCESS = (
     "channel while the bot is running (or send the @username if it's public), then forward a post again."
 )
 
-_REF_RX = re.compile(r"(?i)^(?:https?://)?(?:t\.me|telegram\.me)/(?:c/(\d+)|([A-Za-z0-9_]{3,32}))(?:/\d+)?/?$")
-
-
-def parse_channel_ref(text: str):
-    """@name / t.me link / numeric id -> username (str) or channel id (int); None if unrecognised."""
-    t = (text or "").strip()
-    m = _REF_RX.match(t)
-    if m:
-        return int(m.group(1)) if m.group(1) else m.group(2)
-    t = t.lstrip("@")
-    if re.fullmatch(r"-?\d+", t):
-        s = str(abs(int(t)))
-        return int(s[3:]) if s.startswith("100") and len(s) > 10 else int(s)
-    if re.fullmatch(r"[A-Za-z0-9_]{3,32}", t):
-        return t
-    return None
-
 
 def rights_line(r) -> str:
     if r is None:
@@ -120,10 +104,9 @@ def register(ctx: Ctx) -> None:
         if not chans:
             raise UserError("No channels yet. Use /addchannel.")
         lines, kb = [], []
-        for ch in chans:
-            r = await get_rights(client, ch)
+        for n, ch in enumerate(chans, 1):
             name = esc(ch.title) + (f" (@{esc(ch.username)})" if ch.username else "")
-            lines.append(f"• <b>{name}</b>\n   {rights_line(r)}")
+            lines.append(f"{n}. <b>{name}</b>")
             if ctx.cfg.is_owner(event.sender_id):
                 kb.append([Button.inline(f"➖ Remove {short(ch.title, 22)}", f"chd:{ch.id}")])
         await say(event, "📢 <b>Your channels</b>\n\n" + "\n".join(lines), kb or None)

app/handlers/masslinks.py

@@ -0,0 +1,159 @@
+"""New links, one button after the other.
+
+* "Mass replace links" in the panel of a published button post: the bot asks for a new link for that post's button and,
+  after each answer, for the next button post of the channel (posts without a link button are skipped).
+* /autopost uses the same steps to ask for the real link of every button it posted.
+
+The position in the list lives in the user's state; each answer changes the button in the channel at once.
+"""
+from __future__ import annotations
+
+import logging
+
+from telethon import Button, errors
+
+from ..buttons import link_positions, norm_url, set_field
+from ..common import Ctx, UserError, clear_state, is_cb, on_cb, on_text, say, set_state, show
+from ..tgutil import build_markup, edit_raw, esc, explain_rpc, flood_retry, peer_of, post_link, short
+from .panel import load_post
+
+log = logging.getLogger(__name__)
+
+MODE = "link_queue"
+HEADINGS = {"mass": "🔗 <b>Mass replace links</b>", "auto": "🔗 <b>Links for the new posts</b>"}
+
+
+async def set_link(ctx: Ctx, post, ch, r: int, c: int, url: str):
+    """Point button (r, c) of a published post at `url`: the buttons of the channel message change (only the buttons,
+    the text stays as it is), then the saved post follows."""
+    rows = set_field(post.buttons, r, c, u=url)
+    if post.status == "sent" and post.message_id:
+        try:
+            await flood_retry(lambda: edit_raw(ctx.client, peer_of(ch), post.message_id, markup=build_markup(rows)))
+        except errors.MessageNotModifiedError:
+            pass  # it already had that link
+        except errors.RPCError as e:
+            raise UserError("❌ " + explain_rpc(e) + "\nThe link of this button was NOT changed.")
+    return await ctx.db.update_post(post.id, buttons=rows)
+
+
+def mass_items(posts) -> list:
+    """[(post id, row, column)] of every link button in `posts`, in the order given."""
+    return [[p.id, r, c] for p in posts for r, c in link_positions(p.buttons)]
+
+
+async def current_target(ctx: Ctx, st: dict):
+    """(post, channel, row, column) the list is at, moving past buttons that are gone; None when the list is finished."""
+    while st["i"] < len(st["items"]):
+        pid, r, c = st["items"][st["i"]]
+        post = await ctx.db.get_post(pid)
+        if post is not None and post.status == "sent" and post.message_id and (r, c) in link_positions(post.buttons):
+            ch = await ctx.db.get_channel(post.channel_id)
+            if ch is not None:
+                return post, ch, r, c
+        st["i"] += 1
+        st["gone"] += 1
+    return None
+
+
+def summary(st: dict, stopped: bool) -> str:
+    left = max(0, len(st["items"]) - st["i"]) if stopped else 0
+    bits = [f"{st['done']} link(s) replaced"]
+    if st["skipped"]:
+        bits.append(f"{st['skipped']} skipped")
+    if st["gone"]:
+        bits.append(f"{st['gone']} button(s) no longer there")
+    if left:
+        bits.append(f"{left} not asked")
+    return ("Stopped. " if stopped else "✅ <b>All done.</b> ") + ", ".join(bits) + "."
+
+
+def prompt_for(st: dict, post, ch, r: int, c: int) -> str:
+    total = len(st["items"])
+    b = post.buttons[r][c]
+    lines = [f"{HEADINGS[st['kind']]} · <b>{esc(ch.title)}</b> ({st['i'] + 1} of {total})", ""]
+    where = f"Post #{post.id}"
+    if post.message_id:
+        where = f'<a href="{post_link(ch, post.message_id)}">Post #{post.id}</a>'
+    several = len(link_positions(post.buttons))
+    label = f"“{esc(short(b.get('t') or '', 60))}”"
+    lines.append(f"{where} · button {label}" + (f" (button {r + 1}.{c + 1} of {several} links)" if several > 1 else ""))
+    lines.append(f"Now: <code>{esc(b.get('u') or '')}</code>")
+    lines += ["", "🔗 Send the new link for this button."]
+    return "\n".join(lines)
+
+
+async def ask(ctx: Ctx, event, st: dict, lead: str = "", *, new: bool = True) -> None:
+    """Ask for the next link, or finish. `lead` (what just happened) goes in front, in the same message."""
+    tgt = await current_target(ctx, st)
+    kb_done = [[Button.inline("📚 My posts", "pl")]]
+    send = say if (new or not is_cb(event)) else show
+    if tgt is None:
+        clear_state(ctx, event.sender_id)
+        await send(event, (lead + "\n\n" if lead else "") + summary(st, False), kb_done)
+        return
+    post, ch, r, c = tgt
+    kb = [[Button.inline("⏭ Skip this one", f"lqs:{st['i']}"), Button.inline("✖️ Cancel", "lqx")]]
+    await send(event, (lead + "\n\n" if lead else "") + prompt_for(st, post, ch, r, c), kb)
+
+
+async def begin_queue(ctx: Ctx, event, *, kind: str, cid: int, items: list, lead: str = "", new: bool = True) -> None:
+    """Start asking: `items` is [(post id, row, column)] in the order the links are wanted."""
+    st = set_state(ctx, event.sender_id, MODE, kind=kind, cid=cid, items=[list(i) for i in items], i=0, done=0, skipped=0, gone=0)
+    await ask(ctx, event, st, lead, new=new)
+
+
+def register(ctx: Ctx) -> None:
+    db = ctx.db
+
+    # ------------------------------------------------------------- the panel option
+    @on_cb(ctx, "mlp")
+    async def cb_mass_start(event, parts):
+        post, ch = await load_post(ctx, parts[0])
+        if post.status != "sent" or not post.message_id:
+            raise UserError("Mass replace works on published posts only.")
+        if not link_positions(post.buttons):
+            raise UserError("This post has no button with a link.")
+        posts = await db.sent_posts_from(ch.id, post.message_id)
+        await begin_queue(ctx, event, kind="mass", cid=ch.id, items=mass_items(posts), new=False)
+
+    # ---------------------------------------------------------------- the answers
+    @on_text(ctx, MODE)
+    async def h_link(event, st):
+        url = norm_url(event.raw_text or "")
+        if not url:
+            raise UserError("That doesn't look like a link. Send something like https://t.me/yourbot?start=abc")
+        tgt = await current_target(ctx, st)
+        if tgt is None:  # everything this list was about has vanished meanwhile
+            await ask(ctx, event, st)
+            return
+        post, ch, r, c = tgt
+        label = post.buttons[r][c].get("t") or ""
+        await set_link(ctx, post, ch, r, c, url)
+        st["i"] += 1
+        st["done"] += 1
+        await ask(ctx, event, st, f"✅ <b>{esc(short(label, 40))}</b> now opens <code>{esc(url)}</code>")
+
+    def live(event, parts):
+        st = ctx.state.get(event.sender_id)
+        if not st or st.get("mode") != MODE:
+            raise UserError("That list is no longer active.")
+        if parts and parts[0].isdigit() and int(parts[0]) != st["i"]:
+            raise UserError("That button belongs to an earlier step - use the newest message.")
+        return st
+
+    @on_cb(ctx, "lqs")
+    async def cb_skip(event, parts):
+        st = live(event, parts)
+        st["i"] += 1
+        st["skipped"] += 1
+        await ask(ctx, event, st, "⏭ Skipped.", new=False)
+
+    @on_cb(ctx, "lqx")
+    async def cb_stop(event, parts):
+        st = ctx.state.get(event.sender_id)
+        if not st or st.get("mode") != MODE:
+            await show(event, "Cancelled.")
+            return
+        clear_state(ctx, event.sender_id)
+        await show(event, summary(st, True), [[Button.inline("📚 My posts", "pl")]])

app/handlers/posts.py

@@ -367,6 +367,7 @@ def register(ctx: Ctx) -> None:
             sent = await send_post(client, peer_of(ch), post)
         except errors.RPCError as e:
             raise UserError("❌ " + explain_rpc(e) + "\nThe post was NOT published.")
+        await db.release_slot(ch.id, sent.id, keep_pid=post.id)  # the sync may have picked the message up a moment ago
         post = await db.update_post(post.id, status="sent", message_id=sent.id, sent_at=utcnow())
         await show(
             event,
@@ -413,5 +414,7 @@ def register(ctx: Ctx) -> None:
                     ctx.userbot, bot_error=out.bot_error, fallback_error=out.fallback_error, tried=out.tried_userbot
                 )
                 raise UserError(f"Telegram did not delete that message. {why}\nDelete it by hand in the channel, then use “Only forget it in the bot”.")
+        if post.status == "sent" and post.message_id and parts[1] == "b":
+            ctx.sync_ignore.add((post.channel_id, post.message_id))  # forgotten on purpose: the sync must not bring it back
         await db.delete_post(post.id)
         await show(event, "🗑 Done.", [[Button.inline("📚 My posts", "pl")]])

app/handlers/replace.py

@@ -1,4 +1,4 @@
-"""/replace and /undo (owner only)."""
+"""/replace, /handleswap and /undo (owner only)."""
 from __future__ import annotations
 
 import logging
@@ -13,7 +13,7 @@ from ..common import Ctx, UserError, cmd, edit_callback_message, guard, on_cb, p
 from ..linkswap import normalize_username
 from ..replace_engine import ApplyResult, ChannelScan, ScanOptions, apply_channel, scan_channel, undo_batch
 from ..tgutil import esc, explain_rpc, get_rights
-from .channels import parse_channel_ref
+from ..channelref import parse_channel_ref
 
 log = logging.getLogger(__name__)
 
@@ -25,6 +25,15 @@ USAGE = (
 )
 
 
+USAGE_HANDLES = (
+    "Usage: <code>/handleswap @old @new</code>\n"
+    "Finds the posts that mention the username <code>@old</code> (in their text or caption, and in button names) in all "
+    "channels and changes it to <code>@new</code>. Only usernames change: links, hyperlinks and button links are left alone.\n"
+    "Options: <code>--channel @name</code> (only that channel), <code>--last 200</code> (only the newest 200 post ids)."
+)
+HANDLE_PREFIX = "hs"  # ids of /handleswap runs start with this, so /undo can word its question right
+
+
 @dataclass
 class Job:
     user: int
@@ -34,7 +43,7 @@ class Job:
     created: float = field(default_factory=time.monotonic)
 
 
-def parse_replace_args(raw: str) -> tuple:
+def parse_replace_args(raw: str, *, handles: bool = False) -> tuple:
     toks = (raw or "").split()
     flags = {"channel": None, "last": None, "posts": False, "typed": None}
     pos: list = []
@@ -49,10 +58,10 @@ def parse_replace_args(raw: str) -> tuple:
                 raise ValueError("--last needs a number, for example --last 200")
             flags["last"] = int(toks[i + 1])
             i += 2
-        elif t == "--posts":
+        elif t == "--posts" and not handles:
             flags["posts"] = True
             i += 1
-        elif t == "--no-typed":
+        elif t == "--no-typed" and not handles:
             flags["typed"] = False
             i += 1
         elif t.startswith("--"):
@@ -80,7 +89,43 @@ def pick_channels(chans: list, ref) -> list:
     return out
 
 
+def handles_summary(job: Job) -> str:
+    o = job.opts
+    lines = [f"🔎 <b>Scan finished</b> - swap the username <code>@{esc(o.old)}</code> → <code>@{esc(o.new)}</code>"]
+    total = 0
+    for s in job.scans:
+        head = f"\n<b>{esc(s.channel.title)}</b>"
+        if s.error:
+            lines.append(f"{head}\n⚠️ {esc(s.error)}")
+            continue
+        block = f"{head} - looked at post ids {s.first}-{s.scanned}"
+        if s.total == 0:
+            block += "\nNothing to change."
+        else:
+            t = sum(i.typed for i in s.infos.values())
+            b = sum(i.buttons for i in s.infos.values())
+            what = f"{t} mention(s) in the text" + (f", {b} in button names" if b else "")
+            block += f"\n{s.total} post(s) to change: {what}"
+            if s.foreign:
+                block += f"\n• {s.foreign} of them were not posted by this bot"
+            cb = sum(1 for i in s.infos.values() if i.callbacks)
+            if cb:
+                block += f"\n• {cb} carry other bots' buttons (e.g. reactions) - those stay as they are"
+        if s.skipped_post_links:
+            block += f"\n• left {s.skipped_post_links} mention(s) that are part of a link alone"
+        if s.not_editable:
+            block += f"\n⚠️ {s.not_editable} post(s) by others can't be edited: the bot lacks “Edit messages of others” here"
+        total += s.total
+        lines.append(block)
+    lines.append(f"\n<b>Total: {total} post(s).</b>")
+    if total:
+        lines.append("Nothing has been changed yet. Every edit is logged, so /undo can restore the old versions.")
+    return "\n".join(lines)
+
+
 def summary_text(job: Job) -> str:
+    if job.opts.mode == "handles":
+        return handles_summary(job)
     o = job.opts
     lines = [f"🔎 <b>Scan finished</b> - replace <code>@{esc(o.old)}</code> → <code>@{esc(o.new)}</code>"]
     total = 0
@@ -154,8 +199,6 @@ def register(ctx: Ctx) -> None:
             raise UserError(f"{e}\n\n" + re.sub(r"<[^>]+>", "", USAGE))
         if not chans:
             raise UserError("No channels registered yet. Use /addchannel.")
-        if lock.locked():
-            raise UserError("Another /replace or /undo is still running - wait for it to finish.")
         opts = ScanOptions(
             old=old,
             new=new,
@@ -163,6 +206,30 @@ def register(ctx: Ctx) -> None:
             include_posts=flags["posts"],
             last=flags["last"],
         )
+        await begin_scan(event, opts, chans)
+
+    # ---------------------------------------------------------------- /handleswap
+    @client.on(cmd("handleswap", args=True))
+    @guard(ctx, owner=True)
+    async def h_handleswap(event):
+        purge_pending(ctx)
+        raw = (event.pattern_match.group(1) or "").strip()
+        if not raw:
+            await say(event, USAGE_HANDLES)
+            return
+        try:
+            old, new, flags = parse_replace_args(raw, handles=True)
+            chans = pick_channels(await db.list_channels(), flags["channel"])
+        except ValueError as e:
+            raise UserError(f"{e}\n\n" + re.sub(r"<[^>]+>", "", USAGE_HANDLES))
+        if not chans:
+            raise UserError("No channels registered yet. Use /addchannel.")
+        await begin_scan(event, ScanOptions(old=old, new=new, last=flags["last"], mode="handles"), chans)
+
+    async def begin_scan(event, opts, chans) -> None:
+        """Read the channels, show what would change and wait for the owner's yes."""
+        if lock.locked():
+            raise UserError("Another /replace, /handleswap or /undo is still running - wait for it to finish.")
         async with lock:
             status = await say(event, f"🔍 Scanning {len(chans)} channel(s)…")
             last_edit = [0.0]
@@ -223,8 +290,8 @@ def register(ctx: Ctx) -> None:
             raise UserError("It is already running.")
         job.running = True
         upd = make_updater(event)
-        batch_id = secrets.token_hex(6)
         o = job.opts
+        batch_id = (HANDLE_PREFIX if o.mode == "handles" else "") + secrets.token_hex(6)
         totals = ApplyResult()
         stopped: list = []
         async with lock:
@@ -255,7 +322,8 @@ def register(ctx: Ctx) -> None:
                 if res.aborted:
                     stopped.append((s.channel.title, res.aborted))
         ctx.pending.pop(token, None)
-        lines = [f"✅ <b>Done</b> - <code>@{esc(o.old)}</code> → <code>@{esc(o.new)}</code>", f"• edited: <b>{totals.edited}</b>"]
+        what = "username swap " if o.mode == "handles" else ""
+        lines = [f"✅ <b>Done</b> - {what}<code>@{esc(o.old)}</code> → <code>@{esc(o.new)}</code>", f"• edited: <b>{totals.edited}</b>"]
         if totals.unchanged:
             lines.append(f"• already fine / changed meanwhile: {totals.unchanged}")
         if totals.missing:
@@ -267,10 +335,14 @@ def register(ctx: Ctx) -> None:
             lines.append(f"• <b>failed: {len(totals.failed)}</b> ({sample}{'…' if len(totals.failed) > 8 else ''})")
             names = {n for _, n in totals.failed}
             if names & {"MessageAuthorRequiredError", "InlineBotRequiredError", "ChatAdminRequiredError", "MessageIdInvalidError"}:
-                lines.append("  Telegram doesn't let this bot edit those posts. Use /repost to post them again with the new links.")
+                lines.append(
+                    "  Telegram doesn't let this bot edit those posts."
+                    + ("" if o.mode == "handles" else " Use /repost to post them again with the new links.")
+                )
         for title, name in stopped:
             lines.append(f"⚠️ Stopped early in <b>{esc(title)}</b>: Telegram refused 5 edits in a row ({esc(name)}).")
-        kb = [[Button.inline("↩️ Undo this replace", f"ruc:{batch_id}")]] if totals.edited else None
+        undo_label = "↩️ Undo this swap" if o.mode == "handles" else "↩️ Undo this replace"
+        kb = [[Button.inline(undo_label, f"ruc:{batch_id}")]] if totals.edited else None
         if kb:
             try:
                 await edit_callback_message(event, "\n".join(lines), kb)
@@ -282,9 +354,10 @@ def register(ctx: Ctx) -> None:
     # -------------------------------------------------------------------- /undo
     async def ask_undo(event, batch) -> None:
         n = await db.count_batch_changes(batch.id)
+        what = "username swap" if batch.id.startswith(HANDLE_PREFIX) else "replace"
         await say(
             event,
-            f"↩️ Undo the replace <code>@{esc(batch.old_username)}</code> → <code>@{esc(batch.new_username)}</code>?\n"
+            f"↩️ Undo the {what} <code>@{esc(batch.old_username)}</code> → <code>@{esc(batch.new_username)}</code>?\n"
             f"{n} post(s) will be put back exactly as they were before.",
             [[Button.inline("✅ Yes, undo", f"ru:{batch.id}"), Button.inline("✖️ No", "cx")]],
         )
@@ -294,7 +367,7 @@ def register(ctx: Ctx) -> None:
     async def h_undo(event):
         batch = await db.last_batch()
         if batch is None:
-            raise UserError("There is no /replace run to undo.")
+            raise UserError("There is nothing to undo - no /replace or /handleswap run yet.")
         await ask_undo(event, batch)
 
     @on_cb(ctx, "ruc")
@@ -309,7 +382,7 @@ def register(ctx: Ctx) -> None:
     async def cb_undo(event, parts):
         require_owner(ctx, event)
         if lock.locked():
-            raise UserError("Another /replace or /undo is still running.")
+            raise UserError("Another /replace, /handleswap or /undo is still running.")
         upd = make_updater(event)
         async with lock:
             await upd("⏳ Restoring…", force=True)

app/handlers/repost.py

@@ -11,7 +11,8 @@ from telethon import Button, errors
 
 from ..common import Ctx, UserError, cmd, edit_callback_message, guard, on_cb, purge_pending, require_owner, say
 from ..linkswap import normalize_username
-from ..repost_engine import RepostOptions, RepostPlan, delete_copies, plan_repost, run_repost
+from ..postlinks import resolve_chain
+from ..repost_engine import RepostOptions, RepostPlan, delete_copies, plan_repost, relink_copies, run_repost
 from ..tgutil import esc, get_rights, post_link, short
 from ..userbot import delete_problem_text
 
@@ -107,16 +108,27 @@ def plan_text(p: RepostPlan, o: RepostOptions, title: str, delay: float) -> str:
         )
     if p.dropped_posts:
         lines.append(f"• {p.dropped_posts} post(s) carry other bots' buttons (reactions...) - those are not copied")
+    if p.post_links:
+        lines.append(
+            f"• {p.post_links} link(s) in {p.post_link_posts} post(s) point at other posts of this channel: "
+            "they are pointed at the new copies (a link to a post that doesn't exist stays as it is)"
+        )
+    if p.replies:
+        lines.append(f"• {p.replies} post(s) answer another post: each copy answers the copy of that post")
+    if p.pinned:
+        lines.append(
+            f"• {p.pinned} pinned post(s): their copies are pinned at the end "
+            "(the bot needs the “Edit messages of others” right for that)"
+        )
     minutes = max(1, round(p.eta_seconds(delay) / 60))
     lines += [
         "",
-        "<b>How:</b> posts without buttons or links to change are copied by Telegram itself, so they stay exactly as "
-        "they are. Posts with buttons (or links to swap) are posted again with their text, formatting, media and "
-        "buttons in one go, and each one is read back to make sure its buttons are really there. Copies go to the end "
-        "of the channel silently, in the original order, and are registered in My posts.",
+        "<b>How:</b> posts without buttons, replies or links to change are copied by Telegram itself, so they stay "
+        "exactly as they are. Posts with buttons, links to change or a reply are posted again with their text, "
+        "formatting, media and buttons in one go, and each one is read back to make sure its buttons are really "
+        "there. Copies go to the end of the channel silently, in the original order, and are registered in My posts.",
         "",
-        "<b>Not kept:</b> original dates, view counts, reactions, comments, poll votes, pinned state and links to the "
-        "old posts.",
+        "<b>Not kept:</b> original dates, view counts, reactions, comments and poll votes.",
         f"Time: about {minutes} min. Please don't post in the channel meanwhile.",
         "<b>Nothing is deleted in this step.</b>",
     ]
@@ -125,6 +137,43 @@ def plan_text(p: RepostPlan, o: RepostOptions, title: str, delay: float) -> str:
     return "\n".join(lines)
 
 
+def finishing_notes(res) -> list:
+    """Lines about answers, links and pins for the end of a repost."""
+    notes = []
+    if res.replies:
+        notes.append(f"↩️ {res.replies} post(s) answer the copy of the post they answered.")
+    if res.reply_lost:
+        notes.append(f"{res.reply_lost} post(s) could only be copied without their reply (their media can't be rebuilt).")
+    if res.reply_dropped:
+        notes.append(f"{res.reply_dropped} post(s) answered a post that is not there any more, so they answer nothing.")
+    fin = res.final
+    if fin is not None:
+        if fin.relinked:
+            notes.append(f"🔗 {fin.links} link(s) in {fin.relinked} post(s) now point at the new copies.")
+        if fin.failed:
+            sample = ", ".join(f"#{i} {esc(n)}" for i, n in fin.failed[:5])
+            notes.append(f"{len(fin.failed)} post(s) could not get their links updated ({sample}) - /fixlinks tries again.")
+        if fin.pinned:
+            notes.append(f"📌 {fin.pinned} pinned post(s) are pinned again.")
+        if fin.pin_error:
+            notes.append(
+                f"📌 {fin.pin_wanted - fin.pinned} copy/copies could not be pinned ({esc(fin.pin_error)}). Give the bot the "
+                "“Edit messages of others” right, or pin them by hand."
+            )
+    if res.final_error:
+        notes.append(f"Links and pins could not be finished ({esc(res.final_error)}). /fixlinks repairs the links.")
+    return notes
+
+
+FIXLINKS_TEXT = (
+    "🔗 <b>Fix links to reposted posts</b>\n"
+    "After a repost, a link such as <code>t.me/channel/115</code> has to point at the copy of post 115, because the "
+    "old post is gone. /repost does that by itself now. Use this for a channel you reposted with an earlier version: "
+    "it looks at the copies and fixes every link that still points at a deleted original.\n\n"
+    "Which channel?"
+)
+
+
 def register(ctx: Ctx) -> None:
     client, db, cfg = ctx.client, ctx.db, ctx.cfg
 
@@ -272,17 +321,17 @@ def register(ctx: Ctx) -> None:
         last = [0.0]
 
         async def prog(res):
-            if time.monotonic() - last[0] < 4:
+            if res.phase == "copy" and time.monotonic() - last[0] < 4:
                 return
             last[0] = time.monotonic()
             done = res.copied_units + res.skipped_done
             of = f"/{total}" if total else ""
+            if res.phase == "finish":
+                body = f"🔗 Copied {done}{of} posts. Now pointing links at the new copies and pinning…"
+            else:
+                body = f"⏳ Copying <b>{esc(ch.title)}</b>: {done}{of} posts…\nThe old posts stay untouched."
             try:
-                await edit_callback_message(
-                    event,
-                    f"⏳ Copying <b>{esc(ch.title)}</b>: {done}{of} posts…\nThe old posts stay untouched.",
-                    [[Button.inline("⏹ Stop", f"rps:{mig.id}")]],
-                )
+                await edit_callback_message(event, body, [[Button.inline("⏹ Stop", f"rps:{mig.id}")]])
             except errors.MessageNotModifiedError:
                 pass
             except Exception:
@@ -317,6 +366,7 @@ def register(ctx: Ctx) -> None:
                 )
         if res.aborted:
             notes.append(f"Stopped early because of {esc(res.aborted)}.")
+        notes += finishing_notes(res)
         await edit_callback_message(event, text + ("\n\n" + "\n".join(notes) if notes else ""), kb)
 
     @on_cb(ctx, "rpa")
@@ -500,3 +550,84 @@ def register(ctx: Ctx) -> None:
         mig, _ = await load(parts)
         await db.set_migration_status(mig.id, "closed")
         await edit_callback_message(event, "✅ Closed. Nothing else will be changed in that channel by this repost.")
+
+    # ------------------------------------------------------------------- /fixlinks
+    async def fix_scope(cid: int):
+        """The channel and the links to repair: posts whose original was deleted and whose copy exists."""
+        ch = await db.get_channel(cid)
+        if ch is None:
+            raise UserError("That channel is not registered any more.")
+        pairs = await db.migration_pairs(channel_id=ch.id, only_deleted_old=True)
+        return ch, pairs
+
+    @client.on(cmd("fixlinks"))
+    @guard(ctx, owner=True)
+    async def h_fixlinks(event):
+        chans = await db.list_channels()
+        if not chans:
+            raise UserError("No channels registered yet. Use /addchannel.")
+        kb = [[Button.inline(f"📢 {short(c.title, 40)}", f"fxch:{c.id}")] for c in chans]
+        kb.append([Button.inline("✖️ Cancel", "fxn")])
+        await say(event, FIXLINKS_TEXT, kb)
+
+    @on_cb(ctx, "fxn")
+    async def cb_fix_cancel(event, parts):
+        require_owner(ctx, event)
+        await edit_callback_message(event, "Cancelled - nothing was changed.")
+
+    @on_cb(ctx, "fxch")
+    async def cb_fix_scan(event, parts):
+        require_owner(ctx, event)
+        ch, pairs = await fix_scope(int(parts[0]))
+        if not pairs:
+            await edit_callback_message(
+                event,
+                f"Nothing to fix in <b>{esc(ch.title)}</b>: no repost with deleted originals is recorded for it.",
+            )
+            return
+        if ctx.lock.locked():
+            raise UserError("Another long job is still running.")
+        async with ctx.lock:
+            await edit_callback_message(event, f"🔍 Looking at {len(pairs)} copied post(s) of <b>{esc(ch.title)}</b>…")
+            res = await relink_copies(client, db, ch, resolve_chain(dict(pairs)), [n for _, n in pairs], dry_run=True)
+        if not res.relinked:
+            await edit_callback_message(
+                event,
+                f"✅ <b>{esc(ch.title)}</b>: {res.checked} copied post(s) checked - no link points at a deleted original.",
+            )
+            return
+        await edit_callback_message(
+            event,
+            f"🔗 <b>{esc(ch.title)}</b>: {res.links} link(s) in {res.relinked} post(s) still point at posts that were "
+            f"reposted and deleted. They will point at the copies instead. ({res.checked} copied post(s) checked)",
+            [[Button.inline(f"🔗 Fix {res.links} link(s)", f"fxgo:{ch.id}"), Button.inline("✖️ Cancel", "fxn")]],
+        )
+
+    @on_cb(ctx, "fxgo")
+    async def cb_fix_go(event, parts):
+        require_owner(ctx, event)
+        ch, pairs = await fix_scope(int(parts[0]))
+        if ctx.lock.locked():
+            raise UserError("Another long job is still running.")
+        last = [0.0]
+
+        async def prog(r):
+            if time.monotonic() - last[0] < 3:
+                return
+            last[0] = time.monotonic()
+            try:
+                await edit_callback_message(event, f"🔗 Fixing links in <b>{esc(ch.title)}</b>… {r.relinked} post(s) done")
+            except Exception:
+                pass
+
+        async with ctx.lock:
+            res = await relink_copies(
+                client, db, ch, resolve_chain(dict(pairs)), [n for _, n in pairs], delay=cfg.edit_delay, progress=prog
+            )
+        text = f"✅ <b>{esc(ch.title)}</b>: {res.links} link(s) in {res.relinked} post(s) now point at the copies."
+        if res.failed:
+            sample = ", ".join(f"#{i} {esc(n)}" for i, n in res.failed[:5])
+            text += f"\n⚠️ {len(res.failed)} post(s) failed ({sample}). Run /fixlinks again to retry them."
+        if res.aborted:
+            text += f"\nStopped early because of {esc(res.aborted)}."
+        await edit_callback_message(event, text)

app/handlers/shift.py

@@ -0,0 +1,374 @@
+"""/shift <source> <destination> [from_id] [to_id]: copy posts from one channel into another. Owner only."""
+from __future__ import annotations
+
+import logging
+import re
+import secrets
+import time
+from dataclasses import dataclass, field
+from typing import Optional
+
+from telethon import Button, errors
+
+from ..channelref import ChannelRef, resolve_channel
+from ..common import Ctx, UserError, cmd, edit_callback_message, guard, on_cb, purge_pending, require_owner, say
+from ..repost_engine import RepostOptions, RepostPlan, plan_repost
+from ..shift_engine import delete_shift_copies, run_shift
+from ..tgutil import esc, explain_rpc, get_rights, post_link
+from ..userbot import delete_problem_text
+from .repost import finishing_notes
+
+log = logging.getLogger(__name__)
+
+USAGE = (
+    "Usage: <code>/shift &lt;source&gt; &lt;destination&gt; [from_id] [to_id]</code>\n"
+    "Copies posts from one channel into another - text, formatting, media, albums, buttons, replies - and registers "
+    "the copies in My posts. The source is only read: nothing in it is changed or deleted.\n\n"
+    "A channel can be given as <code>@username</code>, a t.me link, its id (<code>-100…</code>) or an invite link "
+    "(the bot must be a member). The bot must be an admin of the destination (Post messages).\n\n"
+    "<code>/shift @source @destination</code> - every post\n"
+    "<code>/shift @source @destination 15 20</code> - only the posts with message ids 15 to 20\n"
+    "<code>/shift @source @destination 15</code> - only post 15"
+)
+
+
+@dataclass
+class ShiftJob:
+    """A shift that was planned and is waiting for the owner's OK (or is running)."""
+
+    user: int
+    src: ChannelRef
+    dst: ChannelRef
+    plan: RepostPlan
+    already: int = 0  # posts of the range that an earlier shift copied into this destination
+    running: bool = False
+    stop: bool = False
+    shift_id: str = ""
+    created: float = field(default_factory=time.monotonic)
+
+
+def parse_shift_args(raw: str) -> tuple:
+    """-> (source text, destination text, first id or None, last id or None)"""
+    toks = (raw or "").split()
+    if len(toks) < 2 or len(toks) > 4:
+        raise ValueError("Give the source and the destination channel, and optionally the first and the last message id.")
+    nums: list = []
+    for t in toks[2:]:
+        if not t.isdigit() or int(t) < 1:
+            raise ValueError(f"“{t}” is not a message id. Message ids are positive numbers, for example 15 20.")
+        nums.append(int(t))
+    first = last = None
+    if len(nums) == 1:
+        first = last = nums[0]
+    elif len(nums) == 2:
+        first, last = min(nums), max(nums)
+    return toks[0], toks[1], first, last
+
+
+def plan_text(p: RepostPlan, src: ChannelRef, dst: ChannelRef, already: int, delay: float) -> str:
+    kinds = f"{p.text} text, {p.media} media"
+    if p.polls:
+        kinds += f", {p.polls} poll(s)"
+    if p.other:
+        kinds += f", {p.other} other"
+    lines = [
+        "📦 <b>Shift plan</b>",
+        f"From <b>{esc(src.title)}</b> → <b>{esc(dst.title)}</b>",
+        f"Post ids {p.first}-{p.last}: <b>{p.units} posts</b> to copy ({p.messages} messages, {p.albums} album(s)) - {kinds}",
+    ]
+    if p.service:
+        lines.append(f"• {p.service} service message(s) (like “pinned a message”) can't be copied and are skipped")
+    if p.polls:
+        lines.append("• polls are copied without their votes")
+    if p.dropped_posts:
+        lines.append(f"• {p.dropped_posts} post(s) carry other bots' buttons (reactions...) - those are not copied")
+    if p.post_links:
+        lines.append(
+            f"• {p.post_links} link(s) in {p.post_link_posts} post(s) point at other posts of {esc(src.title)}: those that "
+            "are shifted too will point at their copies"
+        )
+    if p.replies:
+        lines.append(f"• {p.replies} post(s) answer another post: the copy answers the copy (if that post is shifted too)")
+    if already:
+        lines.append(
+            f"• ⚠️ {already} of these posts were already shifted into {esc(dst.title)} before: they would be copied again"
+        )
+    minutes = max(1, round(p.eta_seconds(delay) / 60))
+    lines += [
+        "",
+        "<b>How:</b> posts are copied by Telegram itself where possible (exactly as they are); posts with buttons or "
+        "replies are posted again with their text, formatting, media and buttons in one go, and every button is read "
+        "back to make sure it is there. Copies go to the end of the destination silently, in the original order, and "
+        "are registered in My posts.",
+        "",
+        "<b>Not kept:</b> original dates, view counts, reactions, comments, poll votes and pins.",
+        f"Time: about {minutes} min. <b>Nothing is changed in {esc(src.title)}.</b>",
+    ]
+    return "\n".join(lines)
+
+
+def register(ctx: Ctx) -> None:
+    client, db, cfg = ctx.client, ctx.db, ctx.cfg
+
+    # ---------------------------------------------------------------- views
+    async def view_for(sh):
+        """Text and buttons describing where a shift stands."""
+        dst = await db.get_channel(sh.dst_channel_id)
+        c = await db.shift_counts(sh.id)
+        src_name = esc(sh.src_title or str(sh.src_channel_id))
+        dst_name = esc(dst.title) if dst else str(sh.dst_channel_id)
+        route = f"<b>{src_name}</b> → <b>{dst_name}</b>"
+        first_copy = ""
+        if dst and c["first_new"]:
+            first_copy = f'\n<a href="{post_link(dst, c["first_new"])}">First copy</a>'
+        undo = [Button.inline("↩️ Undo - remove the copies", f"sfu:{sh.id}")]
+        if sh.status == "copied":
+            text = (
+                f"✅ {route}: <b>{c['copied']} message(s) copied</b> (source ids {sh.first_id}-{sh.last_id}) and "
+                f"registered in My posts.{first_copy}\nThe source was not changed. Look at the end of the destination; "
+                "if something is wrong, undo removes just these copies."
+            )
+            kb = [undo]
+        elif sh.status == "copies_deleted":
+            text = f"↩️ {route}: the copies were removed again."
+            kb = []
+        elif sh.status == "stopped":
+            text = f"⏹ {route}: <b>stopped</b>. {c['copied']} message(s) copied so far.{first_copy}"
+            kb = [[Button.inline("▶️ Continue", f"sfc:{sh.id}")], undo]
+        elif sh.status == "copying":
+            text = f"⏳ {route}: a shift was running when the bot stopped. {c['copied']} message(s) copied.{first_copy}"
+            kb = [[Button.inline("▶️ Continue", f"sfc:{sh.id}")], undo]
+        else:  # incomplete
+            text = (
+                f"⚠️ {route}: <b>not everything was copied</b>. {c['copied']} message(s) copied.{first_copy}\n"
+                "Nothing was changed in the source."
+            )
+            kb = [[Button.inline("🔁 Try the rest again", f"sfc:{sh.id}")], undo]
+        return text, kb
+
+    # --------------------------------------------------------------- /shift
+    @client.on(cmd("shift", args=True))
+    @guard(ctx, owner=True)
+    async def h_shift(event):
+        purge_pending(ctx)
+        raw = (event.pattern_match.group(1) or "").strip()
+        if not raw:
+            sh = await db.open_shift()
+            if sh is not None:
+                text, kb = await view_for(sh)
+                await say(event, "Unfinished shift:\n\n" + text, kb)
+            else:
+                await say(event, USAGE)
+            return
+        try:
+            src_text, dst_text, first, last = parse_shift_args(raw)
+        except ValueError as e:
+            raise UserError(f"{e}\n\n" + re.sub(r"<[^>]+>", "", USAGE).replace("&lt;", "<").replace("&gt;", ">"))
+        src = await resolve_channel(ctx, src_text)
+        dst = await resolve_channel(ctx, dst_text)
+        if src.id == dst.id:
+            raise UserError("The source and the destination are the same channel. To rebuild one channel in order, use /repost.")
+        sh = await db.open_shift(src.id, dst.id)
+        if sh is not None:
+            text, kb = await view_for(sh)
+            await say(event, "There is an unfinished shift between these two channels. Finish or undo it first.\n\n" + text, kb)
+            return
+        rights = await get_rights(client, dst)
+        if rights is not None and not (rights.admin and rights.post):
+            raise UserError(f"The bot needs the “Post messages” admin right in {dst.title}.")
+        if ctx.lock.locked():
+            raise UserError("Another long job (replace / repost / shift) is still running.")
+        note = await say(event, f"🔍 Reading <b>{esc(src.title)}</b>…")
+        last_edit = [0.0]
+
+        async def prog(p):
+            if time.monotonic() - last_edit[0] < 3:
+                return
+            last_edit[0] = time.monotonic()
+            try:
+                await note.edit(f"🔍 Reading <b>{esc(src.title)}</b>: post id {p.scanned}…")
+            except Exception:
+                pass
+
+        try:
+            async with ctx.lock:
+                plan = await plan_repost(client, src, RepostOptions(), progress=prog, first_id=first, last_id=last)
+        except errors.RPCError as e:
+            raise UserError(
+                f"I can't read {src.title}: {explain_rpc(e)}\nThe bot has to be a member of the source channel "
+                "(an admin is best; it only reads)."
+            )
+        if plan.error:
+            await note.edit(f"⚠️ {esc(plan.error)}")
+            return
+        already = len(await db.shifted_ids(src.id, dst.id, plan.first, plan.last))
+        token = secrets.token_hex(4)
+        ctx.pending[token] = ShiftJob(user=event.sender_id, src=src, dst=dst, plan=plan, already=already)
+        kb = [[Button.inline("▶️ Start copying", f"sfa:{token}"), Button.inline("✖️ Cancel", f"sfn:{token}")]]
+        await note.edit(plan_text(plan, src, dst, already, cfg.edit_delay), buttons=kb)
+
+    @on_cb(ctx, "sfn")
+    async def cb_cancel(event, parts):
+        require_owner(ctx, event)
+        ctx.pending.pop(parts[0], None)
+        await edit_callback_message(event, "Cancelled - nothing was copied.")
+
+    # ------------------------------------------------------------------ copy
+    async def run_copy(event, sid: str, job: Optional[ShiftJob] = None) -> None:
+        sh = await db.get_shift(sid)
+        dst = await db.get_channel(sh.dst_channel_id)
+        if dst is None:
+            raise UserError("The destination channel is not registered any more.")
+        job = job or ShiftJob(user=event.sender_id, src=None, dst=None, plan=None)  # type: ignore[arg-type]
+        job.running, job.stop, job.shift_id = True, False, sid
+        ctx.pending[f"job:{sid}"] = job
+        total = job.plan.units if job.plan else None
+        last = [0.0]
+        names = f"<b>{esc(sh.src_title)}</b> → <b>{esc(dst.title)}</b>"
+
+        async def prog(res):
+            if res.phase == "copy" and time.monotonic() - last[0] < 4:
+                return
+            last[0] = time.monotonic()
+            done = res.copied_units + res.skipped_done
+            of = f"/{total}" if total else ""
+            if res.phase == "finish":
+                body = f"🔗 Copied {done}{of} posts. Now pointing links at the new copies…"
+            else:
+                body = f"⏳ Shifting {names}: {done}{of} posts…\nThe source stays untouched."
+            try:
+                await edit_callback_message(event, body, [[Button.inline("⏹ Stop", f"sfs:{sid}")]])
+            except errors.MessageNotModifiedError:
+                pass
+            except Exception:
+                log.debug("progress update failed", exc_info=True)
+
+        try:
+            async with ctx.lock:
+                await db.set_shift_status(sid, "copying")
+                res = await run_shift(
+                    client, db, sh, dst, event.sender_id, delay=cfg.edit_delay, progress=prog, should_stop=lambda: job.stop
+                )
+        finally:
+            job.running = False
+        status = "stopped" if res.stopped else ("incomplete" if (res.failed or res.aborted) else "copied")
+        await db.set_shift_status(sid, status)
+        sh = await db.get_shift(sid)
+        text, kb = await view_for(sh)
+        notes = []
+        if res.copied_units:
+            notes.append(f"{res.forwarded} post(s) were copied by Telegram, {res.rebuilt} were posted again with their buttons.")
+        if res.repaired:
+            notes.append(f"{res.repaired} post(s) needed their buttons set again after posting - they show them now.")
+        if res.failed:
+            sample = ", ".join(f"#{i} {esc(n)}" for i, n in res.failed[:6])
+            notes.append(f"<b>{len(res.failed)} post(s) failed</b> ({sample}{'…' if len(res.failed) > 6 else ''}).")
+            if any(n == "ButtonsNotShown" for _, n in res.failed):
+                notes.append(
+                    "ButtonsNotShown: Telegram did not show the buttons on the new post, so that copy was removed again. "
+                    "The original is untouched - use “Try the rest again”."
+                )
+            if any(n == "ReplyParentMissing" for _, n in res.failed):
+                notes.append("ReplyParentMissing: that post answers a post whose copy failed; it follows when you try again.")
+        if res.aborted:
+            notes.append(f"Stopped early because of {esc(res.aborted)}.")
+        notes += finishing_notes(res)
+        await edit_callback_message(event, text + ("\n\n" + "\n".join(notes) if notes else ""), kb)
+
+    @on_cb(ctx, "sfa")
+    async def cb_start(event, parts):
+        require_owner(ctx, event)
+        job = ctx.pending.get(parts[0])
+        if not isinstance(job, ShiftJob) or job.user != event.sender_id:
+            raise UserError("That plan has expired. Run /shift again.")
+        if job.running or ctx.lock.locked():
+            raise UserError("A long job is already running.")
+        if await db.open_shift(job.src.id, job.dst.id) is not None:
+            raise UserError("There is already an unfinished shift between these two channels.")
+        rights = await get_rights(client, job.dst)
+        if rights is not None and not (rights.admin and rights.post):
+            raise UserError(f"The bot needs the “Post messages” admin right in {job.dst.title}.")
+        # the copies are registered in My posts, so the destination has to be one of the bot's channels
+        await db.save_channel(job.dst.id, job.dst.access_hash, job.dst.title, job.dst.username, event.sender_id)
+        sid = secrets.token_hex(6)
+        p = job.plan
+        await db.create_shift(sid, src=job.src, dst_channel_id=job.dst.id, first_id=p.first, last_id=p.last, user_id=event.sender_id)
+        ctx.pending.pop(parts[0], None)
+        await run_copy(event, sid, job)
+
+    @on_cb(ctx, "sfc")
+    async def cb_continue(event, parts):
+        require_owner(ctx, event)
+        sh = await db.get_shift(parts[0])
+        if sh is None or sh.status not in ("stopped", "incomplete", "copying"):
+            raise UserError("There is nothing to continue.")
+        if ctx.lock.locked():
+            raise UserError("A long job is already running.")
+        await run_copy(event, sh.id)
+
+    @on_cb(ctx, "sfs")
+    async def cb_stop(event, parts):
+        require_owner(ctx, event)
+        job = ctx.pending.get(f"job:{parts[0]}")
+        if isinstance(job, ShiftJob) and job.running:
+            job.stop = True
+            await event.respond("⏹ Stopping after the post that is being copied…")
+        else:
+            await event.respond("Nothing is running.")
+
+    # ------------------------------------------------------------------ undo
+    async def load(parts):
+        sh = await db.get_shift(parts[0])
+        if sh is None:
+            raise UserError("I don't know that shift any more.")
+        dst = await db.get_channel(sh.dst_channel_id)
+        if dst is None:
+            raise UserError("The destination channel is not registered any more.")
+        return sh, dst
+
+    @on_cb(ctx, "sfb")
+    async def cb_back(event, parts):
+        sh, _ = await load(parts)
+        text, kb = await view_for(sh)
+        await edit_callback_message(event, text, kb)
+
+    @on_cb(ctx, "sfu")
+    async def cb_undo_ask(event, parts):
+        require_owner(ctx, event)
+        sh, dst = await load(parts)
+        c = await db.shift_counts(sh.id)
+        if not c["new_left"]:
+            raise UserError("There is nothing to remove.")
+        await edit_callback_message(
+            event,
+            f"↩️ <b>Remove the {c['new_left']} copied message(s) from {esc(dst.title)}?</b>\n"
+            f"The posts in {esc(sh.src_title)} are not touched.",
+            [[Button.inline("↩️ Yes, remove the copies", f"sfuy:{sh.id}"), Button.inline("Back", f"sfb:{sh.id}")]],
+        )
+
+    @on_cb(ctx, "sfuy")
+    async def cb_undo(event, parts):
+        require_owner(ctx, event)
+        sh, dst = await load(parts)
+        if ctx.lock.locked():
+            raise UserError("A long job is already running.")
+        fallback, deleter = await ctx.userbot.fallback_for(dst) if ctx.userbot is not None else (None, None)
+        try:
+            async with ctx.lock:
+                res = await delete_shift_copies(client, db, dst, sh, delay=0.5, fallback=fallback)
+        finally:
+            if deleter is not None:
+                await deleter.close()
+        if res.remaining == 0 and not res.error:
+            await db.set_shift_status(sh.id, "copies_deleted")
+            await edit_callback_message(
+                event, f"↩️ Done. {res.deleted} copied message(s) removed from {esc(dst.title)}; the source was not touched."
+            )
+        else:
+            why = esc(delete_problem_text(ctx.userbot, bot_error=res.error, fallback_error=res.fallback_error, tried=res.tried_userbot))
+            await edit_callback_message(
+                event,
+                f"⚠️ Removed {res.deleted}, {res.remaining} copy/copies are still there. {why}",
+                [[Button.inline("🔁 Try again", f"sfuy:{sh.id}")]],
+            )
+

app/handlers/sync.py

@@ -0,0 +1,193 @@
+"""/sync (owner only) and the live watch over the channels.
+
+The live part listens to Telegram's updates: a post deleted, edited or added in a connected channel is recorded in My
+posts a few seconds later, and service messages ("pinned a message", "channel photo changed" ...) are deleted at once.
+The same comparison runs on a timer for everything the bot was not there to see (SYNC_INTERVAL_MINUTES).
+"""
+from __future__ import annotations
+
+import logging
+import re
+import time
+from typing import Optional
+
+from telethon import events, types
+
+from ..common import Ctx, UserError, cmd, guard, say
+from ..sync_engine import SyncReport
+from ..syncer import Syncer
+from ..tgutil import esc
+from .replace import pick_channels
+
+log = logging.getLogger(__name__)
+
+USAGE = (
+    "Usage: <code>/sync</code>\n"
+    "Compares every channel with My posts right now: posts that were deleted or edited in the channel by someone else "
+    "are updated here, new posts that were not made with this bot are added, and service messages (“pinned a "
+    "message”, “channel photo changed” ...) that appeared meanwhile are deleted.\n"
+    "Options: <code>--channel @name</code> (only that channel), <code>--clean</code> (also delete the service messages "
+    "that are already in the channel's history; the helper account, if you set one up, deletes the old ones the bot may not)."
+)
+
+
+def parse_sync_args(raw: str) -> tuple:
+    """(channel reference or None, clean) - ValueError with a message for anything else."""
+    toks = (raw or "").split()
+    ref: Optional[str] = None
+    clean = False
+    i = 0
+    while i < len(toks):
+        t = toks[i]
+        if t == "--channel" and i + 1 < len(toks):
+            ref = toks[i + 1]
+            i += 2
+        elif t == "--clean":
+            clean = True
+            i += 1
+        else:
+            raise ValueError(f"I don't know “{t}”.")
+    return ref, clean
+
+
+def ids_text(ids, limit: int = 8) -> str:
+    ids = sorted(ids)
+    shown = ", ".join(str(i) for i in ids[:limit])
+    return shown + (f" and {len(ids) - limit} more" if len(ids) > limit else "")
+
+
+def channel_block(rep: SyncReport) -> str:
+    lines = [f"<b>{esc(rep.channel.title)}</b>"]
+    if rep.error:
+        return "\n".join(lines + [f"⚠️ {esc(rep.error)}"])
+    nothing = True
+    if rep.first_look:
+        lines.append(
+            "👀 First look at this channel. From now on, posts made, edited or deleted outside this bot are recorded "
+            "here (posts from before are not imported)."
+        )
+    if rep.checked:
+        lines.append(f"✓ {rep.checked} saved post(s) compared with the channel")
+    if rep.deleted:
+        nothing = False
+        lines.append(f"🗑 {len(rep.deleted)} deleted in the channel → removed from My posts (message {ids_text(rep.deleted)})")
+    if rep.edited:
+        nothing = False
+        lines.append(f"✏️ {len(rep.edited)} edited in the channel → updated in My posts (message {ids_text(rep.edited)})")
+    if rep.adopted:
+        nothing = False
+        lines.append(f"➕ {len(rep.adopted)} new post(s) made outside the bot → added to My posts (message {ids_text(rep.adopted)})")
+    if rep.refreshed:
+        lines.append(f"• {rep.refreshed} saved post(s) tidied up (spaces at the ends of the text)")
+    if rep.services_deleted:
+        nothing = False
+        lines.append(f"🧹 {rep.services_deleted} service message(s) deleted")
+    if rep.services_failed:
+        nothing = False
+        lines.append(
+            f"⚠️ {rep.services_failed} service message(s) could not be deleted ({esc(rep.services_error or 'refused')}) - "
+            "the bot needs the “Delete messages” right, and Telegram lets bots delete only recent messages"
+        )
+    if rep.buttons_missing:
+        nothing = False
+        lines.append(
+            f"⚠️ {len(rep.buttons_missing)} saved post(s) show no buttons in the channel (message {ids_text(rep.buttons_missing)}). "
+            "The saved buttons were kept: /posts → the channel → 🔧 Check buttons puts them back"
+        )
+    if rep.unsupported:
+        lines.append(f"• {rep.unsupported} new message(s) of a kind My posts can't hold (poll, sticker ...) were left alone")
+    fresh = len(rep.young) + len(rep.young_new)
+    if fresh:
+        lines.append(f"⏳ {fresh} too fresh to judge - the next check will see them")
+    if rep.changed_meanwhile:
+        lines.append(f"• {rep.changed_meanwhile} saved post(s) were changed through the bot meanwhile - left as they are")
+    if rep.anomaly:
+        nothing = False
+        lines.append(f"⚠️ {esc(rep.anomaly)}")
+    if rep.busy:
+        lines.append("⏸ A long job started, so this check stopped early.")
+    if nothing and not rep.busy and not rep.first_look:
+        lines.append("Everything matches.")
+    return "\n".join(lines)
+
+
+def report_text(reports: list, *, clean: bool = False) -> str:
+    head = "🔄 <b>Sync finished</b>" + (" (with the service-message clean-up)" if clean else "")
+    if not reports:
+        return head + "\nNothing to look at."
+    return head + "\n\n" + "\n\n".join(channel_block(r) for r in reports)
+
+
+def channel_id_of(msg) -> Optional[int]:
+    peer = getattr(msg, "peer_id", None)
+    return peer.channel_id if isinstance(peer, types.PeerChannel) else None
+
+
+def register(ctx: Ctx) -> None:
+    client, db = ctx.client, ctx.db
+    if ctx.syncer is None:
+        ctx.syncer = Syncer(ctx)
+    sy = ctx.syncer
+
+    # ----------------------------------------------------------------------------------------- /sync
+    @client.on(cmd("sync", args=True))
+    @guard(ctx, owner=True)
+    async def h_sync(event):
+        raw = (event.pattern_match.group(1) or "").strip()
+        try:
+            ref, clean = parse_sync_args(raw)
+            chans = pick_channels(await db.list_channels(), ref)
+        except ValueError as e:
+            raise UserError(f"{e}\n\n" + re.sub(r"<[^>]+>", "", USAGE))
+        if not chans:
+            raise UserError("No channels registered yet. Use /addchannel.")
+        if ctx.lock.locked():
+            raise UserError("Another long job (replace, repost, shift ...) is still running - try again when it has finished.")
+        async with ctx.lock:
+            status = await say(event, f"🔄 Looking at {len(chans)} channel(s)…")
+            last = [0.0]
+
+            async def prog(ch, rep) -> None:
+                now = time.monotonic()
+                if now - last[0] < 3:
+                    return
+                last[0] = now
+                try:
+                    await status.edit(f"🔄 <b>{esc(ch.title)}</b>: {rep.phase} (message {rep.scanned_to})…")
+                except Exception:
+                    pass
+
+            reports = await sy.run_all(chans, explicit=True, clean=clean, progress=prog)
+        await say(event, report_text(reports, clean=clean))
+
+    # ------------------------------------------------------------------- what Telegram tells while we run
+    @client.on(events.Raw(types.UpdateNewChannelMessage))
+    async def on_channel_message(update):
+        try:
+            m = update.message
+            cid = channel_id_of(m)
+            if cid is None:
+                return
+            if isinstance(m, types.MessageService):
+                await sy.on_service(cid, m)
+            elif not isinstance(m, types.MessageEmpty):
+                sy.note_new(cid, m)
+        except Exception:
+            log.warning("could not handle a new channel message", exc_info=True)
+
+    @client.on(events.Raw(types.UpdateEditChannelMessage))
+    async def on_channel_edit(update):
+        try:
+            m = update.message
+            cid = channel_id_of(m)
+            if cid is not None and not isinstance(m, (types.MessageService, types.MessageEmpty)):
+                sy.note_edit(cid, m)
+        except Exception:
+            log.warning("could not handle an edited channel message", exc_info=True)
+
+    @client.on(events.Raw(types.UpdateDeleteChannelMessages))
+    async def on_channel_delete(update):
+        try:
+            await sy.note_deleted(update.channel_id, list(update.messages))
+        except Exception:
+            log.warning("could not handle deleted channel messages", exc_info=True)

app/handleswap.py

@@ -0,0 +1,95 @@
+"""/handleswap: change a username written in a post - "@bro" -> "@sis" - and nothing else.
+
+Only usernames are touched: the words "@bro" in the text or caption (and in button names). Links, hyperlinks and the
+links behind buttons are left exactly as they are, and so is "@bro" when it is part of a link or an e-mail address.
+"""
+from __future__ import annotations
+
+import copy
+import re
+from typing import Optional
+
+from telethon import types
+from telethon.helpers import add_surrogate, del_surrogate
+
+from .linkswap import MsgChange, apply_span_replacements
+from .tgutil import button_url, is_callback_button, with_text
+
+# a "@name" that follows one of these is part of something else: an e-mail (a@bro), a link (t.me/x@bro, ?u=@bro) ...
+_NOT_BEFORE = r"\w@./=?&#%"
+_PROTECTED = tuple(
+    getattr(types, n) for n in ("MessageEntityUrl", "MessageEntityTextUrl", "MessageEntityEmail") if hasattr(types, n)
+)
+
+
+def mention_regex(name: str):
+    """'@name' as a whole word (case does not matter): not '@name_bot', not 'x@name'."""
+    return re.compile(rf"(?<![{_NOT_BEFORE}])@{re.escape(name)}(?![A-Za-z0-9_])", re.I)
+
+
+def find_mention_spans(sur: str, entities, name: str) -> tuple:
+    """([(start, end)], skipped) - where '@name' stands in `sur` (UTF-16 units). Mentions that sit inside a link or a
+    hyperlink are not changed; `skipped` counts them."""
+    guarded = [(e.offset, e.offset + e.length) for e in entities or [] if isinstance(e, _PROTECTED)]
+    spans, skipped = [], 0
+    for m in mention_regex(name).finditer(sur):
+        s, e = m.span()
+        if any(gs < e and s < ge for gs, ge in guarded):
+            skipped += 1
+            continue
+        spans.append((s, e))
+    return spans, skipped
+
+
+def compute_handle_changes(msg, old: str, new: str) -> tuple:
+    """(MsgChange or None, number of mentions skipped because they are part of a link) - same shape as
+    linkswap.compute_message_changes, so the /replace engine can apply it."""
+    text = msg.message or ""
+    ents = list(msg.entities or [])
+    sur = add_surrogate(text)
+    spans, skipped = find_mention_spans(sur, ents, old) if text else ([], 0)
+    new_sur, new_ents = sur, [copy.copy(e) for e in ents]
+    if spans:
+        new_sur, new_ents = apply_span_replacements(sur, ents, [(s, e, "@" + new) for s, e in spans])
+
+    # the names on link buttons (their links stay)
+    new_markup: Optional[types.ReplyInlineMarkup] = None
+    n_btn, has_cb = 0, False
+    mk = msg.reply_markup
+    if isinstance(mk, types.ReplyInlineMarkup):
+        rx = mention_regex(old)
+        rows = []
+        for row in mk.rows:
+            btns = []
+            for b in row.buttons:
+                if button_url(b) is not None:
+                    label, n = rx.subn("@" + new, getattr(b, "text", "") or "")
+                    if n:
+                        btns.append(with_text(b, label))
+                        n_btn += n
+                        continue
+                elif is_callback_button(b):
+                    has_cb = True
+                btns.append(b)
+            rows.append(types.KeyboardButtonRow(btns))
+        if n_btn:
+            new_markup = types.ReplyInlineMarkup(rows)
+
+    if not (spans or n_btn):
+        return None, skipped
+    return (
+        MsgChange(
+            message_id=msg.id,
+            new_text=del_surrogate(new_sur),
+            new_entities=new_ents,
+            new_markup=new_markup,
+            text_changed=bool(spans),
+            entities_changed=bool(spans),
+            n_buttons=n_btn,
+            n_links=0,
+            n_typed=len(spans),
+            mine=bool(getattr(msg, "out", False)),
+            callbacks=has_cb,
+        ),
+        skipped,
+    )

app/postlinks.py

@@ -0,0 +1,222 @@
+"""Links that point at other posts of the same channel (t.me/name/115, t.me/c/123456/115, tg://...&post=115).
+
+When a channel is reposted its posts get new message ids, so a link to post 115 has to point at the copy of post 115.
+Only the digits of the message id change; everything around them (?single, ?comment=3, the visible text) stays.
+Links can sit in three places: a hyperlink behind text, a link typed into the text, and a button.
+"""
+from __future__ import annotations
+
+import copy
+import re
+from dataclasses import dataclass, field
+from typing import Optional
+
+from telethon import types
+from telethon.helpers import add_surrogate, del_surrogate
+
+from .linkswap import apply_span_replacements
+from .tgutil import button_url, with_url
+
+_HOST = r"(?:t\.me|telegram\.me|telegram\.dog)"
+_NAME = r"[A-Za-z][A-Za-z0-9_]{2,31}"
+# https://t.me/name/115   https://t.me/c/123456/115   (also t.me/s/name/115, the web preview)
+_WEB = re.compile(
+    rf"^(?:https?://)?(?:www\.)?{_HOST}/(?:s/)?(?P<seg>c/(?P<cid>\d+)|(?P<name>{_NAME}))/(?P<id>\d+)(?![A-Za-z0-9_])", re.I
+)
+_TYPED = re.compile(
+    rf"(?<![\w@./=?&#-])(?:https?://)?(?:www\.)?{_HOST}/(?:s/)?(?:c/\d+|{_NAME})/\d+(?![A-Za-z0-9_])", re.I
+)
+# tg://resolve?domain=name&post=115   tg://privatepost?channel=123456&post=115
+_TG = re.compile(r"^tg://(?P<kind>resolve|privatepost)\?(?P<query>[^#]*)", re.I)
+_POST_PARAM = re.compile(r"(?<=[?&])post=(\d+)")
+
+
+@dataclass(frozen=True)
+class Ref:
+    """A link to a post: which channel (username in lower case, or the numeric id) and where the id's digits sit."""
+
+    name: Optional[str]
+    cid: Optional[int]
+    msg_id: int
+    start: int  # the digits of the message id ...
+    end: int
+    seg_start: Optional[int] = None  # ... and the channel part in front of them ("name" or "c/123"); web links only
+
+
+def parse_ref(url: str) -> Optional[Ref]:
+    """What `url` points at, if it is a link to a post of some channel; None for every other link."""
+    url = url or ""
+    m = _WEB.match(url)
+    if m:
+        return Ref(
+            name=m.group("name").lower() if m.group("name") else None,
+            cid=int(m.group("cid")) if m.group("cid") else None,
+            msg_id=int(m.group("id")),
+            start=m.start("id"),
+            end=m.end("id"),
+            seg_start=m.start("seg"),
+        )
+    m = _TG.match(url)
+    if m:
+        params = dict(p.split("=", 1) for p in m.group("query").split("&") if "=" in p)
+        post = _POST_PARAM.search(url)
+        if post is None:
+            return None
+        if m.group("kind").lower() == "resolve" and re.fullmatch(r"[A-Za-z0-9_]+", params.get("domain", "")):
+            return Ref(params["domain"].lower(), None, int(post.group(1)), post.start(1), post.end(1))
+        if m.group("kind").lower() == "privatepost" and params.get("channel", "").isdigit():
+            return Ref(None, int(params["channel"]), int(post.group(1)), post.start(1), post.end(1))
+    return None
+
+
+@dataclass
+class PostLinks:
+    """The links that point into one channel, and where its posts went: `ids` maps old message id -> new one.
+
+    By default the copies are in the same channel (/repost): only the message id changes. With `target_id` the
+    copies are in another channel (/shift): the channel part of the link changes too."""
+
+    username: Optional[str]
+    channel_id: int
+    ids: dict = field(default_factory=dict)
+    target_username: Optional[str] = None
+    target_id: Optional[int] = None
+
+    @classmethod
+    def for_channel(cls, ch, ids: Optional[dict] = None) -> "PostLinks":
+        return cls(username=(ch.username or None), channel_id=int(ch.id), ids=ids if ids is not None else {})
+
+    # -------------------------------------------------------------------------------------- one url
+    def ref_of(self, url: str) -> Optional[Ref]:
+        """The reference in `url` if it points at a post of THIS channel."""
+        r = parse_ref(url)
+        if r is None:
+            return None
+        if r.name is not None and self.username and r.name == self.username.lower():
+            return r
+        if r.cid is not None and r.cid == self.channel_id:
+            return r
+        return None
+
+    def edit_of(self, url: str) -> Optional[tuple]:
+        """(start, end, new text) to apply to `url` so it points at the copy of its post; None if it stays as it is."""
+        r = self.ref_of(url)
+        if r is None or r.msg_id not in self.ids:
+            return None
+        new_id = str(self.ids[r.msg_id])
+        if self.target_id is None:
+            return r.start, r.end, new_id
+        if r.seg_start is None:  # tg:// links are not moved to another channel
+            return None
+        channel = self.target_username or f"c/{self.target_id}"
+        return r.seg_start, r.end, f"{channel}/{new_id}"
+
+    def remap_url(self, url: str) -> str:
+        e = self.edit_of(url)
+        return url if e is None else url[: e[0]] + e[2] + url[e[1] :]
+
+    # --------------------------------------------------------------------------------- whole message
+    def count(self, text: str, entities, markup) -> int:
+        """How many links in a message point at a post of this channel (whether or not the post has a new id)."""
+        n = 0
+        sur = add_surrogate(text or "")
+        for m in _TYPED.finditer(sur):
+            if self.ref_of(m.group(0)):
+                n += 1
+        for e in entities or []:
+            if isinstance(e, types.MessageEntityTextUrl) and self.ref_of(e.url):
+                n += 1
+        if isinstance(markup, types.ReplyInlineMarkup):
+            for row in markup.rows:
+                for b in row.buttons:
+                    u = button_url(b)
+                    if u is not None and self.ref_of(u):
+                        n += 1
+        return n
+
+    def remap_text(self, text: str, entities) -> tuple:
+        """-> (text, entities, typed_links_changed, hyperlinks_changed). Offsets of all entities stay right."""
+        ents = list(entities or [])
+        sur = add_surrogate(text or "")
+        repls = []
+        for m in _TYPED.finditer(sur):
+            e = self.edit_of(m.group(0))
+            if e is not None:
+                repls.append((m.start() + e[0], m.start() + e[1], e[2]))
+        if repls:
+            new_sur, new_ents = apply_span_replacements(sur, ents, repls)
+        else:
+            new_sur, new_ents = sur, [copy.copy(e) for e in ents]
+        hyper = 0
+        for e in new_ents:
+            if isinstance(e, types.MessageEntityTextUrl):
+                new_url = self.remap_url(e.url)
+                if new_url != e.url:
+                    e.url = new_url
+                    hyper += 1
+        return del_surrogate(new_sur), new_ents, len(repls), hyper
+
+    def remap_markup(self, markup) -> tuple:
+        """-> (markup, buttons_changed); the keyboard object is only rebuilt when a button changes."""
+        if not isinstance(markup, types.ReplyInlineMarkup):
+            return markup, 0
+        rows, changed = [], 0
+        for row in markup.rows:
+            btns = []
+            for b in row.buttons:
+                u = button_url(b)
+                new = self.remap_url(u) if u is not None else None
+                if new is not None and new != u:
+                    btns.append(with_url(b, new))
+                    changed += 1
+                else:
+                    btns.append(b)
+            rows.append(types.KeyboardButtonRow(btns))
+        return (types.ReplyInlineMarkup(rows) if changed else markup), changed
+
+
+@dataclass
+class Relinked:
+    """A message whose links to other posts of the channel were pointed at the copies."""
+
+    text: str
+    entities: list
+    markup: Optional[types.ReplyInlineMarkup]
+    text_changed: bool  # the text or its entities differ
+    markup_changed: bool
+    n_typed: int
+    n_hyper: int
+    n_buttons: int
+
+    @property
+    def links(self) -> int:
+        return self.n_typed + self.n_hyper + self.n_buttons
+
+
+def relink_message(msg, links: PostLinks) -> Optional[Relinked]:
+    """The new text / entities / buttons of `msg` once its links are remapped; None when nothing changes."""
+    text, entities, n_typed, n_hyper = links.remap_text(msg.message or "", msg.entities)
+    markup, n_buttons = links.remap_markup(msg.reply_markup)
+    if not (n_typed or n_hyper or n_buttons):
+        return None
+    return Relinked(
+        text=text,
+        entities=entities,
+        markup=markup,
+        text_changed=bool(n_typed or n_hyper),
+        markup_changed=bool(n_buttons),
+        n_typed=n_typed,
+        n_hyper=n_hyper,
+        n_buttons=n_buttons,
+    )
+
+
+def resolve_chain(step: dict) -> dict:
+    """Several reposts in a row: {1: 5, 5: 9} -> {1: 9, 5: 9} (a post that moved twice points at its latest copy)."""
+    out = {}
+    for old in step:
+        cur, seen = old, 0
+        while cur in step and seen < 100:
+            cur, seen = step[cur], seen + 1
+        out[old] = cur
+    return out

app/replace_engine.py

@@ -8,6 +8,7 @@ from typing import Any, Awaitable, Callable, Optional
 
 from telethon import errors, functions, types
 
+from .handleswap import compute_handle_changes
 from .linkswap import compute_message_changes
 from .tgutil import (
     build_markup,
@@ -35,6 +36,16 @@ class ScanOptions:
     include_typed: bool = True
     include_posts: bool = False
     last: Optional[int] = None
+    mode: str = "links"  # "links": the username inside t.me links (/replace); "handles": '@name' written in a post (/handleswap)
+
+
+def compute_changes(m, opts: ScanOptions) -> tuple:
+    """What this run would change in message `m`: (MsgChange or None, skipped)."""
+    if opts.mode == "handles":
+        return compute_handle_changes(m, opts.old, opts.new)
+    return compute_message_changes(
+        m, opts.old, opts.new, include_typed=opts.include_typed, include_posts=opts.include_posts
+    )
 
 
 @dataclass
@@ -136,9 +147,7 @@ async def scan_channel(
             found = True
             if isinstance(m, types.MessageService):
                 continue
-            change, skipped = compute_message_changes(
-                m, opts.old, opts.new, include_typed=opts.include_typed, include_posts=opts.include_posts
-            )
+            change, skipped = compute_changes(m, opts)
             scan.skipped_post_links += skipped
             if not change:
                 continue
@@ -211,9 +220,7 @@ async def apply_channel(
                     res.missing += 1
                     continue
                 # re-check against the live message: it may have been edited since the scan
-                change, _ = compute_message_changes(
-                    m, opts.old, opts.new, include_typed=opts.include_typed, include_posts=opts.include_posts
-                )
+                change, _ = compute_changes(m, opts)
                 if not change:
                     res.unchanged += 1
                     continue

app/repost_engine.py

@@ -9,6 +9,13 @@ Every post is copied to the end of the channel (silently), in the same order as
   3. if the channel forbids forwarding, every post is created like in 2.
 Links can be swapped on the way, the new posts are registered in the database, and the old posts are removed
 later in a separate step. Nothing here deletes anything unless asked to (delete_copies).
+
+What a plain copy would lose is put back as well:
+  * a post that answers another post is posted as an answer to the COPY of that post (a reply can only be set
+    when a post is created, so such a post is always created again instead of copied);
+  * a link to another post of the channel (t.me/name/115) is pointed at the copy of that post - at once when the
+    copy already exists, afterwards (finalize_repost) for posts that are copied later;
+  * a post that was pinned has its copy pinned at the end.
 """
 from __future__ import annotations
 
@@ -21,11 +28,13 @@ from telethon import errors, functions, types, utils
 
 from .buttons_sync import ButtonsNotShown, ensure_markup, link_urls
 from .linkswap import compute_message_changes, url_only_markup
+from .postlinks import PostLinks, relink_message
 from .replace_engine import CHUNK, FALLBACK_EMPTY_BATCHES, fetch_messages, get_top_id, is_skippable, preview_flag
 from .tgutil import (
     build_markup,
     classify_media,
     de_entities,
+    edit_raw,
     flood_retry,
     media_ref,
     peer_of,
@@ -72,9 +81,13 @@ class Final:
     n_links: int = 0
     n_typed: int = 0
     dropped: int = 0  # other bots' buttons that are not copied
+    n_post_links: int = 0  # links to other posts of the same channel (whatever their new id will be)
+    n_relinked: int = 0  # of those, how many were already pointed at a copy
 
 
-def final_state(src, opts: RepostOptions) -> Final:
+def final_state(src, opts: RepostOptions, links: Optional[PostLinks] = None) -> Final:
+    """What the copy of `src` should look like. `links` (optional) points links to other posts of the channel at
+    the copies that exist so far (links.ids: old message id -> new one)."""
     change = None
     if opts.swaps:
         change, _ = compute_message_changes(
@@ -87,6 +100,15 @@ def final_state(src, opts: RepostOptions) -> Final:
     markup, dropped = url_only_markup(
         src.reply_markup, opts.old if opts.swaps else None, opts.new if opts.swaps else None, opts.include_posts
     )
+    n_post_links = n_relinked = 0
+    text_changed = bool(change and change.entities_changed)
+    if links is not None:
+        n_post_links = links.count(text, entities, markup)
+        if n_post_links and links.ids:
+            text, entities, n_typed, n_hyper = links.remap_text(text, entities)
+            markup, n_buttons = links.remap_markup(markup)
+            n_relinked = n_typed + n_hyper + n_buttons
+            text_changed = text_changed or bool(n_typed or n_hyper)
     # Send exactly what is saved: go through the stored form that "My posts" also uses to build its edits.
     markup = build_markup(ser_markup(markup)) if markup is not None else None
     entities = de_entities(ser_entities(entities))
@@ -95,14 +117,42 @@ def final_state(src, opts: RepostOptions) -> Final:
         entities=entities,
         markup=markup,
         preview=preview_flag(src),
-        text_changed=bool(change and change.entities_changed),
+        text_changed=text_changed,
         n_buttons=change.n_buttons if change else 0,
         n_links=change.n_links if change else 0,
         n_typed=change.n_typed if change else 0,
         dropped=dropped,
+        n_post_links=n_post_links,
+        n_relinked=n_relinked,
     )
 
 
+# ---------------------------------------------------------------------------- replies and pins
+class ReplyParentMissing(RuntimeError):
+    """A post answers another post whose copy does not exist (yet): copying it now would lose the answer."""
+
+
+def reply_parent(msg) -> Optional[int]:
+    """The message id a post answers (in the same channel), or None."""
+    h = getattr(msg, "reply_to", None)
+    if h is None or getattr(h, "reply_to_peer_id", None) is not None or getattr(h, "forum_topic", None):
+        return None
+    return getattr(h, "reply_to_msg_id", None) or None
+
+
+def new_parent(msg, idmap: dict, seen: set) -> Optional[int]:
+    """The copy of the post `msg` answers. None when it answers nothing, or the post it answered is gone / outside
+    the range being copied. Raises ReplyParentMissing when that post was read in this run but has no copy."""
+    pid = reply_parent(msg)
+    if pid is None:
+        return None
+    if pid in idmap:
+        return idmap[pid]
+    if pid in seen:
+        raise ReplyParentMissing(f"post {msg.id} answers post {pid}, which has not been copied")
+    return None
+
+
 def new_messages_from(result) -> list:
     """The messages created by a send / forward request, oldest first."""
     out = []
@@ -133,6 +183,10 @@ class RepostPlan:
     n_links: int = 0
     n_typed: int = 0
     dropped_posts: int = 0
+    post_link_posts: int = 0  # posts with a link to another post of this channel
+    post_links: int = 0
+    pinned: int = 0  # pinned posts (their copies get pinned)
+    replies: int = 0  # posts that answer another post
     partial: bool = False
     error: Optional[str] = None
 
@@ -156,8 +210,15 @@ async def find_last_id(client, peer, top: int) -> Optional[int]:
 
 
 async def plan_repost(
-    client, ch, opts: RepostOptions, *, progress: Optional[Callable[[RepostPlan], Awaitable[None]]] = None
+    client,
+    ch,
+    opts: RepostOptions,
+    *,
+    progress: Optional[Callable[[RepostPlan], Awaitable[None]]] = None,
+    first_id: Optional[int] = None,
+    last_id: Optional[int] = None,
 ) -> RepostPlan:
+    """Read the channel and count what a copy would do. first_id / last_id limit the range (used by /shift)."""
     peer = peer_of(ch)
     plan = RepostPlan(channel=ch, partial=bool(opts.last))
     top = await get_top_id(client, peer)
@@ -171,7 +232,14 @@ async def plan_repost(
             plan.error = "The channel has no posts."
             return plan
         first, top = max(1, newest - opts.last + 1), newest
+    if first_id:
+        first = max(first, first_id)
+        plan.partial = True
+    if last_id:
+        top = last_id if top is None else min(top, last_id)
+        plan.partial = True
     plan.first = first
+    links = PostLinks.for_channel(ch)
     a, empty_run, prev_gid, max_seen = first, 0, None, 0
     while True:
         if top is not None and a > top:
@@ -202,7 +270,7 @@ async def plan_repost(
                 plan.polls += 1
             else:
                 plan.other += 1
-            f = final_state(m, opts)
+            f = final_state(m, opts, links)
             if f.n_buttons or f.n_links or f.n_typed:
                 plan.link_posts += 1
                 plan.n_buttons += f.n_buttons
@@ -210,6 +278,13 @@ async def plan_repost(
                 plan.n_typed += f.n_typed
             if f.dropped:
                 plan.dropped_posts += 1
+            if f.n_post_links:
+                plan.post_link_posts += 1
+                plan.post_links += f.n_post_links
+            if getattr(m, "pinned", False):
+                plan.pinned += 1
+            if reply_parent(m) is not None:
+                plan.replies += 1
         plan.scanned = hi
         if top is None:
             empty_run = 0 if found else empty_run + 1
@@ -256,11 +331,15 @@ async def delete_ids(client, peer, ids: list) -> None:
         log.warning("could not remove messages %s: %s", ids, type(e).__name__)
 
 
-async def forward_unit(client, peer, unit: list) -> list:
+async def forward_unit(client, src_peer, unit: list, dst_peer=None) -> list:
+    """Let Telegram copy the posts (no "forwarded from" header). `dst_peer` defaults to the same channel."""
     ids = [m.id for m in unit]
+    to_peer = dst_peer if dst_peer is not None else src_peer
     result = await flood_retry(
         lambda: client(
-            functions.messages.ForwardMessagesRequest(from_peer=peer, id=ids, to_peer=peer, silent=True, drop_author=True)
+            functions.messages.ForwardMessagesRequest(
+                from_peer=src_peer, id=ids, to_peer=to_peer, silent=True, drop_author=True
+            )
         )
     )
     return new_messages_from(result)
@@ -270,8 +349,15 @@ def _inverted(m) -> Optional[bool]:
     return True if getattr(m, "invert_media", False) else None
 
 
-async def resend_unit(client, peer, unit: list, finals: list) -> list:
-    """Rebuild a post from its parts (used when copying is not allowed)."""
+def reply_header(reply_to: Optional[int]):
+    """The `reply_to` argument of the send requests: answer message `reply_to` of the same channel."""
+    return types.InputReplyToMessage(reply_to_msg_id=reply_to) if reply_to else None
+
+
+async def resend_unit(client, peer, unit: list, finals: list, reply_to: Optional[int] = None) -> list:
+    """Rebuild a post from its parts (used when copying is not allowed, or the copy must differ from the original).
+    `reply_to`: message id (in the channel `peer`) the new post answers."""
+    reply = reply_header(reply_to)
     if len(unit) > 1:
         singles = [
             types.InputSingleMedia(media=utils.get_input_media(m.media), message=f.text, entities=f.entities or None)
@@ -280,7 +366,7 @@ async def resend_unit(client, peer, unit: list, finals: list) -> list:
 
         def make():
             return functions.messages.SendMultiMediaRequest(
-                peer=peer, multi_media=singles, silent=True, invert_media=_inverted(unit[0])
+                peer=peer, multi_media=singles, silent=True, invert_media=_inverted(unit[0]), reply_to=reply
             )
 
     else:
@@ -296,6 +382,7 @@ async def resend_unit(client, peer, unit: list, finals: list) -> list:
                     no_webpage=True if f.preview is False else None,
                     invert_media=_inverted(m),
                     silent=True,
+                    reply_to=reply,
                 )
 
         else:
@@ -310,6 +397,7 @@ async def resend_unit(client, peer, unit: list, finals: list) -> list:
                     reply_markup=f.markup,
                     invert_media=_inverted(m),
                     silent=True,
+                    reply_to=reply,
                 )
 
     result = await flood_retry(lambda: client(make()))
@@ -326,22 +414,29 @@ class _State:
     forwarded: int = 0
     rebuilt: int = 0
     repaired: int = 0  # new posts whose buttons had to be set again after posting
+    replies: int = 0  # new posts created as an answer to the copy of the post the original answered
+    reply_lost: int = 0  # answers that could only be copied without the reply
 
 
-def is_pure(src, f: Final) -> bool:
-    """True when a server-side copy is already the final post: nothing to swap and no keyboard to rebuild."""
-    return not f.text_changed and src.reply_markup is None
+def is_pure(src, f: Final, reply_to: Optional[int] = None) -> bool:
+    """True when a server-side copy is already the final post: nothing to swap, no keyboard to rebuild and no
+    reply to set (a reply can only be given when a post is created)."""
+    return not f.text_changed and src.reply_markup is None and not reply_to
 
 
-async def copy_unit(client, peer, unit: list, finals: list, st: _State) -> list:
-    pure = all(is_pure(m, f) for m, f in zip(unit, finals))
+async def copy_unit(
+    client, peer, unit: list, finals: list, st: _State, reply_to: Optional[int] = None, dst_peer=None
+) -> list:
+    """Create the copy of one post (or album) at the end of `dst_peer` (default: the same channel `peer`)."""
+    dst = dst_peer if dst_peer is not None else peer
+    pure = all(is_pure(m, f, reply_to) for m, f in zip(unit, finals))
     if st.mode == "forward" and pure:
         try:
-            news = await forward_unit(client, peer, unit)
+            news = await forward_unit(client, peer, unit, dst)
             if len(news) == len(unit):
                 st.forwarded += 1
                 return news
-            await delete_ids(client, peer, [n.id for n in news])
+            await delete_ids(client, dst, [n.id for n in news])
         except errors.ChatForwardsRestrictedError:
             st.mode = "resend"
             log.info("this channel restricts forwarding - rebuilding posts from their parts instead")
@@ -349,8 +444,25 @@ async def copy_unit(client, peer, unit: list, finals: list, st: _State) -> list:
             if type(e).__name__ in FATAL:
                 raise
             log.info("copy failed (%s) - rebuilding the post", type(e).__name__)
-    news = await resend_unit(client, peer, unit, finals)
+    try:
+        news = await resend_unit(client, dst, unit, finals, reply_to)
+    except (TypeError, ValueError, errors.RPCError) as e:
+        # A kind of media a post cannot be rebuilt from (or a reply Telegram refuses). If only the reply made a
+        # rebuild necessary, a plain copy by Telegram still works - without the reply.
+        if isinstance(e, errors.RPCError) and type(e).__name__ in FATAL:
+            raise
+        if reply_to and st.mode == "forward" and all(is_pure(m, f) for m, f in zip(unit, finals)):
+            news = await forward_unit(client, peer, unit, dst)
+            if len(news) != len(unit):
+                await delete_ids(client, dst, [n.id for n in news])
+                raise
+            st.reply_lost += 1
+            st.forwarded += 1
+            return news
+        raise
     st.rebuilt += 1
+    if reply_to:
+        st.replies += 1
     return news
 
 
@@ -373,6 +485,22 @@ async def settle_buttons(client, peer, finals: list, news: list, st: _State, *,
             st.repaired += 1
 
 
+@dataclass
+class FinalizeResult:
+    """The finishing touches after the copies exist: links to other posts, pins."""
+
+    checked: int = 0  # copies read back
+    relinked: int = 0  # copies whose links to other posts now point at the new copies
+    links: int = 0  # links changed in total
+    missing: int = 0  # copies that do not exist any more
+    failed: list = field(default_factory=list)  # [(message id, error name)]
+    pin_wanted: int = 0  # copies of pinned posts that still had to be pinned
+    pinned: int = 0
+    pin_error: Optional[str] = None
+    aborted: Optional[str] = None
+    stopped: bool = False
+
+
 @dataclass
 class RepostResult:
     copied_units: int = 0
@@ -382,10 +510,16 @@ class RepostResult:
     forwarded: int = 0
     rebuilt: int = 0
     repaired: int = 0
+    replies: int = 0  # copies created as an answer to the copy of the post the original answered
+    reply_lost: int = 0  # answers that could only be copied without the reply
+    reply_dropped: int = 0  # answers to a post that no longer exists (nothing to answer)
     aborted: Optional[str] = None
     stopped: bool = False
     first_new: Optional[int] = None
     last_new: Optional[int] = None
+    phase: str = "copy"  # copy | finish
+    final: Optional[FinalizeResult] = None
+    final_error: Optional[str] = None
 
 
 def _rows(unit: list, finals: list, news: list) -> list:
@@ -410,37 +544,49 @@ def _rows(unit: list, finals: list, news: list) -> list:
     return rows
 
 
-async def run_repost(
+@dataclass
+class CopyJob:
+    """Everything the copy loop needs to know; /repost and /shift each fill it in their own way."""
+
+    src_peer: Any
+    dst_peer: Any  # the same as src_peer for /repost
+    opts: RepostOptions
+    links: PostLinks  # links.ids is `idmap`: old message id -> id of its copy, grows as we go
+    first_id: int
+    last_id: int
+    done: set  # old ids that already have a copy
+    idmap: dict
+    record: Callable[[list], Awaitable[None]]  # saves [{old_id, new_id, post}] (and registers the posts)
+
+
+async def copy_loop(
     client,
-    db,
-    ch,
-    mig,
-    user_id: int,
+    job: CopyJob,
     *,
     delay: float = 1.2,
     progress: Optional[Callable[[RepostResult], Awaitable[None]]] = None,
     should_stop: Optional[Callable[[], bool]] = None,
     settle_pause: float = 0.7,
 ) -> RepostResult:
-    """Copy the posts of migration `mig` (ids first_id..last_id) in order; safe to run again to continue."""
-    opts = RepostOptions(
-        old=mig.old_username, new=mig.new_username, include_typed=mig.include_typed, include_posts=mig.include_posts
-    )
-    peer = peer_of(ch)
-    done = await db.migration_done_old_ids(mig.id)
+    """Copy the posts first_id..last_id of `job.src_peer` to the end of `job.dst_peer`, in order.
+    Safe to run again to continue: posts in `job.done` are skipped."""
+    src, dst = job.src_peer, job.dst_peer
+    seen: set = set()  # old ids read in this run (a reply to one of them needs its copy)
     res, st, streak = RepostResult(), _State(), 0
-    async for unit in iter_units(client, peer, mig.first_id, mig.last_id):
+    async for unit in iter_units(client, src, job.first_id, job.last_id):
         if should_stop and should_stop():
             res.stopped = True
             break
         ids = [m.id for m in unit]
-        if all(i in done for i in ids):
+        seen.update(ids)
+        if all(i in job.done for i in ids):
             res.skipped_done += 1
             continue
-        finals = [final_state(m, opts) for m in unit]
+        finals = [final_state(m, job.opts, job.links) for m in unit]
         try:
-            news = await copy_unit(client, peer, unit, finals, st)
-            await settle_buttons(client, peer, finals, news, st, pause=settle_pause)
+            reply_to = new_parent(unit[0], job.idmap, seen)
+            news = await copy_unit(client, src, unit, finals, st, reply_to, dst)
+            await settle_buttons(client, dst, finals, news, st, pause=settle_pause)
         except Exception as e:  # one bad post must not stop the others
             name = type(e).__name__
             log.warning("copying %s failed: %s %s", ids, name, e)
@@ -452,15 +598,19 @@ async def run_repost(
             await asyncio.sleep(delay)
             continue
         try:
-            await db.record_repost(mig.id, ch.id, _rows(unit, finals, news), user_id)
+            await job.record(_rows(unit, finals, news))
         except Exception as e:  # keep the channel and the database consistent
             log.exception("could not save the copy of %s", ids)
-            await delete_ids(client, peer, [n.id for n in news])
+            await delete_ids(client, dst, [n.id for n in news])
             res.failed.append((ids[0], type(e).__name__))
             res.aborted = type(e).__name__
             break
         streak = 0
-        done.update(ids)
+        job.done.update(ids)
+        for old, new in zip(unit, news):
+            job.idmap[old.id] = new.id
+        if reply_parent(unit[0]) is not None and reply_to is None:
+            res.reply_dropped += 1
         res.copied_units += 1
         res.copied_messages += len(unit)
         res.first_new = res.first_new or news[0].id
@@ -469,6 +619,185 @@ async def run_repost(
             await progress(res)
         await asyncio.sleep(delay)
     res.forwarded, res.rebuilt, res.repaired = st.forwarded, st.rebuilt, st.repaired
+    res.replies, res.reply_lost = st.replies, st.reply_lost
+    return res
+
+
+async def run_repost(
+    client,
+    db,
+    ch,
+    mig,
+    user_id: int,
+    *,
+    delay: float = 1.2,
+    progress: Optional[Callable[[RepostResult], Awaitable[None]]] = None,
+    should_stop: Optional[Callable[[], bool]] = None,
+    settle_pause: float = 0.7,
+) -> RepostResult:
+    """Copy the posts of migration `mig` (ids first_id..last_id) in order; safe to run again to continue.
+
+    When every post is copied (not stopped, no fatal error) the finishing touches follow: links to posts that were
+    copied later are pointed at their copies, and the copies of pinned posts are pinned."""
+    opts = RepostOptions(
+        old=mig.old_username, new=mig.new_username, include_typed=mig.include_typed, include_posts=mig.include_posts
+    )
+    peer = peer_of(ch)
+    idmap = dict(await db.migration_pairs(mid=mig.id, alive_only=False))
+    job = CopyJob(
+        src_peer=peer,
+        dst_peer=peer,
+        opts=opts,
+        links=PostLinks.for_channel(ch, idmap),
+        first_id=mig.first_id,
+        last_id=mig.last_id,
+        done=await db.migration_done_old_ids(mig.id),
+        idmap=idmap,
+        record=lambda rows: db.record_repost(mig.id, ch.id, rows, user_id),
+    )
+    res = await copy_loop(client, job, delay=delay, progress=progress, should_stop=should_stop, settle_pause=settle_pause)
+    if not res.stopped and not res.aborted:
+        res.phase = "finish"
+        if progress:
+            await progress(res)
+        try:
+            res.final = await finalize_repost(
+                client, db, ch, mig, delay=delay, settle_pause=settle_pause, should_stop=should_stop
+            )
+        except Exception as e:  # the copies are fine; only the finishing touches failed
+            log.exception("finishing the repost (links, pins) failed")
+            res.final_error = type(e).__name__
+    return res
+
+
+# --------------------------------------------------------------------- links to copies, pins
+async def relink_copies(
+    client,
+    db,
+    ch,
+    ids: dict,
+    targets: list,
+    *,
+    delay: float = 1.2,
+    settle_pause: float = 0.7,
+    dry_run: bool = False,
+    progress: Optional[Callable[[FinalizeResult], Awaitable[None]]] = None,
+    should_stop: Optional[Callable[[], bool]] = None,
+    links: Optional[PostLinks] = None,
+) -> FinalizeResult:
+    """Point every link to a post of this channel at the copy of that post (`ids`: old message id -> new one).
+
+    `targets` are the message ids of `ch` to look at (the copies). Idempotent: a link that already points at a copy
+    is left alone, because only OLD ids are replaced. With dry_run nothing is edited, the result only counts.
+    `links` (optional) says which links to follow; the default is "links to posts of `ch` itself" (/repost)."""
+    res = FinalizeResult()
+    peer = peer_of(ch)
+    links = links if links is not None else PostLinks.for_channel(ch, ids)
+    targets = sorted(set(targets))
+    streak, last_name = 0, None
+    for i in range(0, len(targets), CHUNK):
+        chunk = targets[i : i + CHUNK]
+        for mid, m in zip(chunk, await fetch_messages(client, peer, chunk)):
+            if is_skippable(m):
+                res.missing += 1
+                continue
+            res.checked += 1
+            rl = relink_message(m, links)
+            if rl is None:
+                continue
+            if dry_run:
+                res.relinked += 1
+                res.links += rl.links
+                continue
+            if should_stop and should_stop():
+                res.stopped = True
+                return res
+            try:
+                await flood_retry(
+                    lambda: edit_raw(
+                        client, peer, m.id, text=rl.text, entities=rl.entities, markup=rl.markup, preview=preview_flag(m)
+                    )
+                )
+                if link_urls(rl.markup):
+                    await ensure_markup(client, peer, m.id, rl.markup, pause=settle_pause)
+            except errors.MessageNotModifiedError:
+                continue
+            except Exception as e:  # one bad post must not stop the others
+                name = type(e).__name__
+                log.warning("could not update the links of %s: %s %s", m.id, name, e)
+                res.failed.append((m.id, name))
+                streak = streak + 1 if name == last_name else 1
+                last_name = name
+                if name in FATAL or streak >= ABORT_AFTER:
+                    res.aborted = name
+                    return res
+                continue
+            streak, last_name = 0, None
+            res.relinked += 1
+            res.links += rl.links
+            try:
+                await db.update_post_by_message(
+                    ch.id, m.id, text=rl.text, entities=ser_entities(rl.entities), buttons=ser_markup(rl.markup)
+                )
+            except Exception:  # the channel is right; the saved copy will be corrected by the next sync
+                log.warning("could not save the new links of %s", m.id, exc_info=True)
+            if progress:
+                await progress(res)
+            await asyncio.sleep(delay)
+    return res
+
+
+async def pin_copies(client, ch, pairs: list, res: FinalizeResult, *, delay: float = 1.2) -> None:
+    """Pin the copy of every post that is pinned among the originals (oldest first, so the newest pin is on top).
+
+    The "pinned a message" notices Telegram adds to the channel are deleted again right away."""
+    peer = peer_of(ch)
+    pairs = sorted(pairs)
+    todo: list = []
+    for i in range(0, len(pairs), CHUNK):
+        chunk = pairs[i : i + CHUNK]
+        olds = await fetch_messages(client, peer, [o for o, _ in chunk])
+        news = await fetch_messages(client, peer, [n for _, n in chunk])
+        for (_, new_id), src, cur in zip(chunk, olds, news):
+            if is_skippable(src) or is_skippable(cur):
+                continue
+            if getattr(src, "pinned", False) and not getattr(cur, "pinned", False):
+                todo.append(new_id)
+    res.pin_wanted = len(todo)
+    for new_id in todo:
+        try:
+            out = await flood_retry(
+                lambda: client(functions.messages.UpdatePinnedMessageRequest(peer=peer, id=new_id, silent=True))
+            )
+        except errors.MessageNotModifiedError:
+            continue  # already pinned
+        except errors.RPCError as e:
+            res.pin_error = type(e).__name__
+            return
+        res.pinned += 1
+        await delete_ids(client, peer, [m.id for m in new_messages_from(out) if isinstance(m, types.MessageService)])
+        await asyncio.sleep(delay)
+
+
+async def finalize_repost(
+    client,
+    db,
+    ch,
+    mig,
+    *,
+    delay: float = 1.2,
+    settle_pause: float = 0.7,
+    progress: Optional[Callable[[FinalizeResult], Awaitable[None]]] = None,
+    should_stop: Optional[Callable[[], bool]] = None,
+) -> FinalizeResult:
+    """After the copies exist: links that point at old posts get the copies' ids, copies of pinned posts get pinned."""
+    pairs = sorted(await db.migration_pairs(mid=mig.id))
+    res = await relink_copies(
+        client, db, ch, dict(pairs), [n for _, n in pairs], delay=delay, settle_pause=settle_pause,
+        progress=progress, should_stop=should_stop,
+    )
+    if not res.stopped and not res.aborted:
+        await pin_copies(client, ch, pairs, res, delay=delay)
     return res
 
 
@@ -522,26 +851,22 @@ async def delete_batch(client, peer, ids: list, fallback: Optional[Callable[[lis
     return out
 
 
-async def delete_copies(
+async def delete_pending(
     client,
-    db,
-    ch,
-    mig,
-    which: str,
+    peer,
+    todo: list,
+    mark: Callable[[list], Awaitable[None]],
     *,
     delay: float = 0.5,
     progress: Optional[Callable[[DeleteResult], Awaitable[None]]] = None,
     fallback: Optional[Callable[[list], Awaitable[Any]]] = None,
 ) -> DeleteResult:
-    """which='old': remove the original posts; which='new': remove the copies (undo). Oldest first, 100 per request.
+    """Delete the messages `todo` (oldest first, 100 per request). `mark(ids)` records the ones that are gone.
 
     The bot tries first and the result is always checked by reading the messages back. Whatever the bot could not
     delete (Telegram may refuse old posts) goes to `fallback(ids)` - the userbot - if there is one, and is checked
-    again. A batch the bot cannot delete does not stop the run: newer posts may still be deletable.
-    """
-    peer = peer_of(ch)
+    again. A batch the bot cannot delete does not stop the run: newer posts may still be deletable."""
     res = DeleteResult()
-    todo = await db.migration_pending_ids(mig.id, which, 1_000_000)
     use_fallback = fallback
     for i in range(0, len(todo), 100):
         ids = todo[i : i + 100]
@@ -553,7 +878,7 @@ async def delete_copies(
             use_fallback = None  # it will not work for the next batches either
         left = [x for x in ids if x not in out.gone]
         if out.gone:
-            await db.mark_migration_deleted(mig.id, ch.id, which, sorted(out.gone))
+            await mark(sorted(out.gone))
             res.deleted += len(out.gone)
         if left and out.bot_error:
             res.error = res.error or out.bot_error  # remembered for the report
@@ -562,6 +887,27 @@ async def delete_copies(
         if progress:
             await progress(res)
         await asyncio.sleep(delay)
+    return res
+
+
+async def delete_copies(
+    client,
+    db,
+    ch,
+    mig,
+    which: str,
+    *,
+    delay: float = 0.5,
+    progress: Optional[Callable[[DeleteResult], Awaitable[None]]] = None,
+    fallback: Optional[Callable[[list], Awaitable[Any]]] = None,
+) -> DeleteResult:
+    """which='old': remove the original posts; which='new': remove the copies (undo). Oldest first, 100 per request."""
+    todo = await db.migration_pending_ids(mig.id, which, 1_000_000)
+
+    async def mark(ids: list) -> None:
+        await db.mark_migration_deleted(mig.id, ch.id, which, ids)
+
+    res = await delete_pending(client, peer_of(ch), todo, mark, delay=delay, progress=progress, fallback=fallback)
     counts = await db.migration_counts(mig.id)
     res.remaining = counts["old_left"] if which == "old" else counts["new_left"]
     res.blocked = res.remaining > 0 and not res.error

app/servicemsg.py

@@ -0,0 +1,55 @@
+"""Service messages: the notices Telegram puts into a channel by itself - "channel name changed", "channel photo
+updated", "<admin> pinned a message", "live stream started" ... The bot removes them from every connected channel.
+"""
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass
+from typing import Optional
+
+from telethon import errors, functions, types
+
+from .tgutil import flood_retry
+
+log = logging.getLogger(__name__)
+
+# Telegram refusals that mean "the bot has no right to delete here" (worth telling the owner about)
+RIGHTS_ERRORS = ("ChatAdminRequiredError", "ChatWriteForbiddenError", "ChannelPrivateError", "UserNotParticipantError")
+
+
+def is_service(m) -> bool:
+    return isinstance(m, types.MessageService)
+
+
+def action_label(m) -> str:
+    """'PinMessage', 'ChatEditTitle', 'GroupCall' ... - for the log."""
+    name = type(getattr(m, "action", None)).__name__
+    return name[len("MessageAction"):] if name.startswith("MessageAction") else name
+
+
+@dataclass
+class Removed:
+    deleted: int = 0
+    failed: int = 0
+    error: Optional[BaseException] = None  # the first thing Telegram refused with
+
+    @property
+    def error_name(self) -> Optional[str]:
+        return type(self.error).__name__ if self.error is not None else None
+
+
+async def remove(client, peer, ids: list) -> Removed:
+    """Delete these messages, 100 per request. A refusal is reported in the result, never raised."""
+    out = Removed()
+    ids = sorted(set(ids))
+    for i in range(0, len(ids), 100):
+        part = ids[i : i + 100]
+        try:
+            res = await flood_retry(lambda part=part: client(functions.channels.DeleteMessagesRequest(channel=peer, id=part)))
+        except errors.RPCError as e:
+            out.failed += len(part)
+            out.error = out.error or e
+            continue
+        n = getattr(res, "pts_count", None)
+        out.deleted += len(part) if n is None else min(len(part), int(n))
+    return out

app/shift_engine.py

@@ -0,0 +1,101 @@
+"""/shift: copy posts of one channel into another channel - text, formatting, media, albums, buttons and replies
+- and register the copies in My posts. The source is only read, never changed or deleted.
+
+It is the /repost copy loop pointed at two channels. Links from a shifted post to another post of the source channel
+follow the content: when that post was shifted too, the link points at its copy in the destination.
+"""
+from __future__ import annotations
+
+import logging
+from types import SimpleNamespace
+from typing import Any, Awaitable, Callable, Optional
+
+from .postlinks import PostLinks
+from .repost_engine import (
+    CopyJob,
+    DeleteResult,
+    RepostOptions,
+    RepostResult,
+    copy_loop,
+    delete_pending,
+    relink_copies,
+)
+from .tgutil import peer_of
+
+log = logging.getLogger(__name__)
+
+
+def source_of(shift) -> SimpleNamespace:
+    """The source channel as stored with the shift (it need not be registered in the bot)."""
+    return SimpleNamespace(
+        id=shift.src_channel_id, access_hash=shift.src_access_hash, username=shift.src_username, title=shift.src_title
+    )
+
+
+async def run_shift(
+    client,
+    db,
+    shift,
+    dst,
+    user_id: int,
+    *,
+    delay: float = 1.2,
+    settle_pause: float = 0.7,
+    progress: Optional[Callable[[RepostResult], Awaitable[None]]] = None,
+    should_stop: Optional[Callable[[], bool]] = None,
+) -> RepostResult:
+    """Copy the posts shift.first_id..shift.last_id of the source into `dst` (a registered channel), in order.
+    Safe to run again to continue: posts that already have a copy are skipped."""
+    src = source_of(shift)
+    idmap = dict(await db.shift_pairs(shift.id, alive_only=False))
+    links = PostLinks(
+        username=src.username, channel_id=src.id, ids=idmap, target_username=dst.username, target_id=dst.id
+    )
+    job = CopyJob(
+        src_peer=peer_of(src),
+        dst_peer=peer_of(dst),
+        opts=RepostOptions(),
+        links=links,
+        first_id=shift.first_id,
+        last_id=shift.last_id,
+        done=await db.shift_done_ids(shift.id),
+        idmap=idmap,
+        record=lambda rows: db.record_shift(shift.id, dst.id, rows, user_id),
+    )
+    res = await copy_loop(client, job, delay=delay, progress=progress, should_stop=should_stop, settle_pause=settle_pause)
+    if not res.stopped and not res.aborted:
+        res.phase = "finish"
+        if progress:
+            await progress(res)
+        try:
+            pairs = await db.shift_pairs(shift.id)
+            res.final = await relink_copies(
+                client, db, dst, dict(pairs), [n for _, n in pairs], links=links, delay=delay,
+                settle_pause=settle_pause, should_stop=should_stop,
+            )
+        except Exception as e:  # the copies are fine; only the links between them were not updated
+            log.exception("pointing the links of a shift at the copies failed")
+            res.final_error = type(e).__name__
+    return res
+
+
+async def delete_shift_copies(
+    client,
+    db,
+    dst,
+    shift,
+    *,
+    delay: float = 0.5,
+    progress: Optional[Callable[[DeleteResult], Awaitable[None]]] = None,
+    fallback: Optional[Callable[[list], Awaitable[Any]]] = None,
+) -> DeleteResult:
+    """Undo: remove the copies of this shift from the destination channel (the source is not touched)."""
+    todo = await db.shift_pending_new(shift.id, 1_000_000)
+
+    async def mark(ids: list) -> None:
+        await db.mark_shift_deleted(shift.id, dst.id, ids)
+
+    res = await delete_pending(client, peer_of(dst), todo, mark, delay=delay, progress=progress, fallback=fallback)
+    res.remaining = (await db.shift_counts(shift.id))["new_left"]
+    res.blocked = res.remaining > 0 and not res.error
+    return res

app/sync_engine.py

@@ -0,0 +1,449 @@
+"""Keeps My posts in step with the channels: what admins do outside the bot is noticed and recorded.
+
+What a look at a channel does
+  * every post the bot has saved is read back from Telegram: a post that is gone is removed here, a post that was edited
+    gets its saved copy updated (text, formatting, buttons, media);
+  * messages that appeared since the last look are saved as new posts (source "adopted"). The very first look only
+    notes where the channel stands: this follows changes, it does not import the old posts;
+  * service messages ("channel name changed", "pinned a message", video chat ...) found on the way are deleted.
+
+Safety rules
+  * posts the bot touched a moment ago, and messages that appeared a moment ago, are left for the next look (the bot may
+    still be saving them);
+  * a saved copy is only changed if nobody changed it since it was read (optimistic check in the database);
+  * buttons that are missing in the channel are never removed from the saved copy (Telegram sometimes hides a keyboard
+    for a while - the saved buttons are what "Check buttons" restores from);
+  * if EVERY saved post of a channel looks deleted, nothing is removed: more likely the bot lost access.
+"""
+from __future__ import annotations
+
+import asyncio
+import logging
+from dataclasses import dataclass, field
+from typing import Any, Awaitable, Callable, Optional
+
+from telethon import errors, functions, types
+
+from .db import as_utc, utcnow
+from .replace_engine import fetch_messages
+from .repost_engine import delete_batch
+from .tgutil import classify_media, explain_rpc, flood_retry, media_from_ref, media_info, peer_of, ser_entities, ser_markup
+
+log = logging.getLogger(__name__)
+
+CHUNK = 100  # Telegram returns at most 100 messages per request
+GRACE = 120.0  # seconds a post or message must be old before it is judged
+PAUSE = 0.25  # between two requests of a scan
+SLACK_CAP = 300  # a quick look reads this many ids beyond the most likely end of the new messages
+DEEP_EVERY = 24 * 3600  # the automatic looks read every possible id at least this often
+MIN_FOR_ANOMALY = 5
+MAX_PROBE = 300  # requests spent finding the newest message the first time (30 000 ids)
+
+
+# ----------------------------------------------------------------------------------------------- options
+@dataclass
+class SyncOptions:
+    deep: bool = False  # read every possible id for new messages, not only the likely ones
+    clean: bool = False  # also sweep the whole history for old service messages
+    delete_services: bool = True
+    adopt: bool = True  # save new posts of others
+    grace: float = GRACE
+    pause: float = PAUSE
+    ignore: Any = field(default_factory=set)  # (channel id, message id) of posts the owner told the bot to forget
+    fallback: Optional[Callable[[list], Awaitable[Any]]] = None  # deletes with the userbot (only used by `clean`)
+
+
+@dataclass
+class SyncReport:
+    channel: Any
+    phase: str = ""
+    checked: int = 0  # saved posts compared with the channel
+    deleted: list = field(default_factory=list)  # message ids that were gone: removed from My posts
+    edited: list = field(default_factory=list)  # message ids whose saved copy was brought up to date
+    refreshed: int = 0  # saved copies corrected for trifles (a trailing space ...)
+    adopted: list = field(default_factory=list)  # message ids of new posts that were saved
+    unsupported: int = 0  # new messages of a kind My posts can't show (polls, stickers ...)
+    buttons_missing: list = field(default_factory=list)  # saved buttons that the channel shows no keyboard for
+    services_deleted: int = 0
+    services_failed: int = 0
+    services_error: Optional[str] = None  # name of the refusal
+    young: list = field(default_factory=list)  # saved posts the bot touched a moment ago: judge them later
+    young_new: list = field(default_factory=list)  # new messages that appeared a moment ago: save them later
+    changed_meanwhile: int = 0  # saved posts that were changed through the bot while we looked (left alone)
+    first_look: bool = False
+    busy: bool = False  # a long job started: the look was cut short
+    anomaly: Optional[str] = None
+    error: Optional[str] = None
+    scanned_to: int = 0
+
+    @property
+    def changed(self) -> bool:
+        return bool(self.deleted or self.edited or self.adopted or self.services_deleted)
+
+
+# ------------------------------------------------------------------------------------- comparing a post
+def age(dt, now) -> float:
+    """Seconds since `dt` (never negative); unknown = very old."""
+    if dt is None:
+        return float("inf")
+    return max(0.0, (now - as_utc(dt)).total_seconds())
+
+
+def media_key(media) -> Optional[tuple]:
+    """Which photo / file a message carries (its id - the file reference changes from fetch to fetch)."""
+    if isinstance(media, types.MessageMediaPhoto) and getattr(media, "photo", None):
+        return ("photo", media.photo.id)
+    if isinstance(media, types.MessageMediaDocument) and getattr(media, "document", None):
+        return ("doc", media.document.id)
+    return None
+
+
+def stored_media_key(ref: Optional[str]) -> Optional[tuple]:
+    if not ref:
+        return None
+    try:
+        return media_key(media_from_ref(ref))
+    except Exception:
+        return ("unreadable", ref[:40])  # differs from anything real: the saved reference gets replaced
+
+
+def live_fields(m) -> dict:
+    """The saved-post fields of a message as it is in the channel now."""
+    kind, ref = media_info(m)
+    return {
+        "text": m.message or "",
+        "entities": ser_entities(m.entities),
+        "buttons": ser_markup(m.reply_markup),
+        "media_kind": kind,
+        "media_file_id": ref,
+    }
+
+
+def message_supported(m) -> bool:
+    """Something My posts can show and edit: text, or a photo / video / GIF / audio / file."""
+    return bool((m.message or "").strip()) or classify_media(m) is not None
+
+
+def adopt_fields(m) -> dict:
+    f = live_fields(m)
+    f["link_preview"] = isinstance(getattr(m, "media", None), types.MessageMediaWebPage)
+    return f
+
+
+@dataclass
+class Diff:
+    fields: dict = field(default_factory=dict)  # what to save
+    buttons_missing: bool = False
+    quiet: bool = False  # only a trifle differs (surrounding spaces): fix it without counting it as an edit
+
+
+def diff_post(p, m) -> Diff:
+    """What differs between the saved post `p` and the live message `m`."""
+    live = live_fields(m)
+    fields: dict = {}
+    if (p.text or "") != live["text"]:
+        fields["text"] = live["text"]
+    if list(p.entities or []) != live["entities"]:
+        fields["entities"] = live["entities"]
+    # media: replaced media is taken over; media that seems to have vanished is not (a message can't lose its media)
+    if live["media_kind"] is not None:
+        if p.media_kind != live["media_kind"] or stored_media_key(p.media_file_id) != media_key(m.media):
+            fields["media_kind"], fields["media_file_id"] = live["media_kind"], live["media_file_id"]
+    missing = False
+    if live["buttons"]:
+        if list(p.buttons or []) != live["buttons"]:
+            fields["buttons"] = live["buttons"]
+    elif p.buttons:
+        missing = True
+    quiet = set(fields) == {"text"} and (p.text or "").strip() == live["text"].strip()
+    return Diff(fields, missing, quiet)
+
+
+# --------------------------------------------------------------------------------------- talking to Telegram
+async def channel_pts(client, peer) -> tuple:
+    """(the channel's event counter, None) or (None, why not). The counter is never lower than the newest message id."""
+    try:
+        full = await flood_retry(lambda: client(functions.channels.GetFullChannelRequest(peer)))
+    except errors.RPCError as e:
+        return None, explain_rpc(e)
+    pts = getattr(getattr(full, "full_chat", None), "pts", None)
+    if pts is None:
+        return None, "Telegram did not say how far the channel has come."
+    return int(pts), None
+
+
+def is_gone(m) -> bool:
+    return m is None or isinstance(m, types.MessageEmpty)
+
+
+async def find_top(client, peer, pts: int, floor: int, *, pause: float = PAUSE) -> int:
+    """The newest message id that exists. Only the event counter is known (never lower), so look downwards from it;
+    `floor` (the newest saved post) is where the looking can stop."""
+    hi = pts
+    for _ in range(MAX_PROBE):
+        if hi <= floor:
+            break
+        lo = max(floor + 1, hi - CHUNK + 1)
+        msgs = await fetch_messages(client, peer, list(range(lo, hi + 1)))
+        alive = [m.id for m in msgs if not is_gone(m)]
+        if alive:
+            return max(alive)
+        hi = lo - 1
+        await asyncio.sleep(pause)
+    return max(floor, hi)
+
+
+async def delete_services(client, peer, ids: list, rep: SyncReport, opts: SyncOptions) -> None:
+    """Delete service messages (bot first; `opts.fallback` for what it may not delete) and count what happened."""
+    ids = sorted(set(ids))
+    for i in range(0, len(ids), CHUNK):
+        part = ids[i : i + CHUNK]
+        out = await delete_batch(client, peer, part, opts.fallback)
+        rep.services_deleted += len(out.gone)
+        left = len(part) - len(out.gone)
+        if left:
+            rep.services_failed += left
+            rep.services_error = rep.services_error or out.bot_error or out.fallback_error or "NotDeleted"
+
+
+# ------------------------------------------------------------------------------------- the three steps
+async def _save_edits(db, rep: SyncReport, edits: list, busy) -> None:
+    for p, d in edits:
+        if busy():
+            rep.busy = True
+            return
+        if await db.sync_update_post(p.id, p.updated_at, d.fields):
+            if d.quiet:
+                rep.refreshed += 1
+            else:
+                rep.edited.append(p.message_id)
+        else:
+            rep.changed_meanwhile += 1
+
+
+async def _adopt(db, ch, rep: SyncReport, fresh: list, opts: SyncOptions, busy, now) -> None:
+    """Save messages the bot does not know (ascending ids); too fresh ones are left for the next look."""
+    known: set = set()
+    ids = [m.id for m in fresh]
+    for i in range(0, len(ids), 500):
+        known |= {p.message_id for p in await db.posts_at(ch.id, ids[i : i + 500])}
+    todo = []
+    for m in fresh:
+        if m.id in known or (ch.id, m.id) in opts.ignore:
+            continue
+        if age(m.date, now) < opts.grace:
+            rep.young_new.append(m.id)
+        elif not message_supported(m):
+            rep.unsupported += 1
+        else:
+            todo.append(m)
+    for m in todo:
+        if busy():
+            rep.busy = True
+            return
+        _, created = await db.adopt_message(ch.id, m.id, adopt_fields(m), sent_at=m.date)
+        if created:
+            rep.adopted.append(m.id)
+
+
+async def _compare_saved(client, db, ch, rep: SyncReport, opts: SyncOptions, busy, progress) -> None:
+    """Every saved post against the channel."""
+    rep.phase = "comparing the saved posts"
+    peer = peer_of(ch)
+    posts = await db.sent_posts(ch.id)  # the saved copies are read BEFORE the channel is, never after
+    now = utcnow()
+    judge = []
+    for p in posts:
+        if min(age(p.updated_at, now), age(p.sent_at, now)) < opts.grace:
+            rep.young.append(p.message_id)
+        else:
+            judge.append(p)
+    rep.checked = len(judge)
+    by_id = {p.message_id: p for p in judge}
+    ids = sorted(by_id)
+    gone, edits = [], []
+    for i in range(0, len(ids), CHUNK):
+        if busy():
+            rep.busy = True
+            return
+        chunk = ids[i : i + CHUNK]
+        msgs = await fetch_messages(client, peer, chunk)
+        for mid, m in zip(chunk, msgs):
+            p = by_id[mid]
+            if is_gone(m):
+                gone.append(p)
+            elif isinstance(m, types.MessageService):
+                continue  # not a post any more (cannot happen: ids are never reused)
+            else:
+                d = diff_post(p, m)
+                if d.buttons_missing:
+                    rep.buttons_missing.append(mid)
+                if d.fields:
+                    edits.append((p, d))
+        rep.scanned_to = chunk[-1]
+        if progress:
+            await progress(rep)
+        await asyncio.sleep(opts.pause)
+    if gone:
+        if len(ids) >= MIN_FOR_ANOMALY and len(gone) == len(ids):
+            rep.anomaly = (
+                f"All {len(ids)} saved posts look deleted, so none was removed here - "
+                "check that the bot is still an admin of the channel."
+            )
+        else:
+            gone_ids = [p.message_id for p in gone]
+            await db.forget_posts(ch.id, gone_ids)
+            rep.deleted.extend(gone_ids)
+    await _save_edits(db, rep, edits, busy)
+
+
+async def _sweep_services(client, ch, rep: SyncReport, opts: SyncOptions, pts: int, busy, progress) -> None:
+    """The whole history, once: every service message that is still there goes."""
+    rep.phase = "removing old service messages"
+    peer = peer_of(ch)
+    a = 1
+    while a <= pts:
+        if busy():
+            rep.busy = True
+            return
+        hi = min(a + CHUNK - 1, pts)
+        msgs = await fetch_messages(client, peer, list(range(a, hi + 1)))
+        ids = [m.id for m in msgs if isinstance(m, types.MessageService)]
+        if ids:
+            await delete_services(client, peer, ids, rep, opts)
+        rep.scanned_to = hi
+        if progress:
+            await progress(rep)
+        a = hi + 1
+        await asyncio.sleep(opts.pause)
+
+
+async def _look_for_new(client, db, ch, rep: SyncReport, opts: SyncOptions, state, pts: int, busy, progress) -> None:
+    """Messages that appeared since the last look: new posts are saved, service messages deleted."""
+    rep.phase = "looking for new posts"
+    peer = peer_of(ch)
+    now = utcnow()
+    if state is None:  # the first look: only note where the channel stands
+        floor = await db.max_saved_id(ch.id)
+        top = await find_top(client, peer, pts, floor, pause=opts.pause)
+        rep.first_look, rep.scanned_to = True, top
+        await db.save_sync(ch.id, last_top=top, last_pts=pts, deferred=0, deep=True)
+        return
+    deep = opts.deep or age(state.last_deep, now) > DEEP_EVERY
+    if pts == state.last_pts and not state.deferred and not deep:
+        return  # not a single event since the last look: a quick look would read exactly what it read then
+    delta = max(0, pts - state.last_pts)
+    slack = min(max(0, state.last_pts - state.last_top), SLACK_CAP)
+    soft = pts if deep else min(pts, state.last_top + delta + slack)
+    a, highest, had_live = state.last_top + 1, state.last_top, True
+    fresh, services = [], []
+    while a <= pts:
+        if a > soft and not had_live:
+            break  # past the likely end and the last stretch was empty
+        if busy():
+            rep.busy = True
+            return
+        hi = min(a + CHUNK - 1, pts)
+        msgs = await fetch_messages(client, peer, list(range(a, hi + 1)))
+        had_live = False
+        for m in msgs:
+            if is_gone(m):
+                continue
+            had_live = True
+            highest = max(highest, m.id)
+            if isinstance(m, types.MessageService):
+                services.append(m)
+            else:
+                fresh.append(m)
+        rep.scanned_to = hi
+        if progress:
+            await progress(rep)
+        a = hi + 1
+        await asyncio.sleep(opts.pause)
+    services_ids = [m.id for m in services]
+    if opts.delete_services and services_ids:
+        await delete_services(client, peer, services_ids, rep, opts)
+    if opts.adopt and fresh:
+        await _adopt(db, ch, rep, fresh, opts, busy, now)
+        if rep.busy:
+            return
+    new_top = highest
+    if rep.young_new:  # these have to be looked at again: don't move the mark past them
+        new_top = min(new_top, min(rep.young_new) - 1)
+    await db.save_sync(
+        ch.id, last_top=max(state.last_top, new_top), last_pts=pts, deferred=len(rep.young_new), deep=deep
+    )
+
+
+# ----------------------------------------------------------------------------------------- entry points
+async def sync_channel(client, db, ch, opts: Optional[SyncOptions] = None, *, busy=None, progress=None) -> SyncReport:
+    """One full look at a channel (see the module text). Never raises Telegram's refusals: they end up in `report.error`."""
+    opts = opts or SyncOptions()
+    busy = busy or (lambda: False)
+    rep = SyncReport(channel=ch)
+    pts, err = await channel_pts(client, peer_of(ch))
+    if pts is None:
+        rep.error = err
+        return rep
+    try:
+        state = await db.get_sync(ch.id)
+        await _compare_saved(client, db, ch, rep, opts, busy, progress)
+        if opts.clean and not rep.busy:
+            await _sweep_services(client, ch, rep, opts, pts, busy, progress)
+        if not rep.busy:
+            await _look_for_new(client, db, ch, rep, opts, state, pts, busy, progress)
+    except errors.RPCError as e:
+        rep.error = explain_rpc(e)
+    return rep
+
+
+async def sync_ids(client, db, ch, ids, opts: Optional[SyncOptions] = None, *, adopt_ids=(), busy=None) -> SyncReport:
+    """A look at these messages only - the ones an update told about. `adopt_ids`: the ones that are new posts, so a
+    message the bot does not know may be saved (an edited message the bot does not know is history: left alone)."""
+    opts = opts or SyncOptions()
+    busy = busy or (lambda: False)
+    rep = SyncReport(channel=ch)
+    peer = peer_of(ch)
+    ids = sorted(set(ids))
+    adopt_set = set(adopt_ids)
+    try:
+        rows = {p.message_id: p for p in await db.posts_at(ch.id, ids)}  # saved copies first, the channel second
+        now = utcnow()
+        gone, edits, fresh, services = [], [], [], []
+        for i in range(0, len(ids), CHUNK):
+            if busy():
+                rep.busy = True
+                return rep
+            chunk = ids[i : i + CHUNK]
+            msgs = await fetch_messages(client, peer, chunk)
+            for mid, m in zip(chunk, msgs):
+                row = rows.get(mid)
+                recent = row is not None and min(age(row.updated_at, now), age(row.sent_at, now)) < opts.grace
+                if is_gone(m):
+                    if row is not None and not recent:
+                        gone.append(row)
+                elif isinstance(m, types.MessageService):
+                    services.append(mid)
+                elif row is not None:
+                    if recent:
+                        rep.young.append(mid)
+                        continue
+                    d = diff_post(row, m)
+                    if d.buttons_missing:
+                        rep.buttons_missing.append(mid)
+                    if d.fields:
+                        edits.append((row, d))
+                elif mid in adopt_set:
+                    fresh.append(m)
+        if opts.delete_services and services:
+            await delete_services(client, peer, services, rep, opts)
+        if gone:
+            gone_ids = [p.message_id for p in gone]
+            await db.forget_posts(ch.id, gone_ids)
+            rep.deleted.extend(gone_ids)
+        await _save_edits(db, rep, edits, busy)
+        if opts.adopt and fresh and not rep.busy:
+            await _adopt(db, ch, rep, fresh, opts, busy, now)
+    except errors.RPCError as e:
+        rep.error = explain_rpc(e)
+    return rep

app/syncer.py

@@ -0,0 +1,288 @@
+"""The background side of the sync.
+
+* a timer: every SYNC_INTERVAL_MINUTES each connected channel is compared with My posts (app/sync_engine.py);
+* a queue for what Telegram tells the bot while it runs: a post deleted / edited / added in a channel is looked at a few
+  seconds later (the wait lets a burst of changes settle, and lets the bot finish saving its own posts first);
+* service messages ("pinned a message", "channel photo changed" ...) are deleted the moment they appear.
+
+Nothing here holds the bot's one-long-job lock: a running /repost, /shift, /replace ... always wins, and what happens
+while it runs is picked up by the next timed check.
+"""
+from __future__ import annotations
+
+import asyncio
+import logging
+import time
+from typing import Optional
+
+from .servicemsg import RIGHTS_ERRORS, action_label, remove
+from .sync_engine import SyncOptions, SyncReport, sync_channel, sync_ids
+from .tgutil import esc, peer_of
+
+log = logging.getLogger(__name__)
+
+
+class Syncer:
+    def __init__(self, ctx):
+        self.ctx = ctx
+        # timings (seconds) - the tests make them small
+        self.start_delay = 90.0  # after the bot starts, before the first timed check
+        self.debounce = 5.0  # quiet time needed before the queue is worked off
+        self.max_debounce = 30.0  # ... but never wait longer than this
+        self.lock_wait = 30.0  # a long job runs: look again after this
+        self.rt_grace = 10.0  # live changes: posts the bot touched / messages younger than this are judged next round
+        self.pause = 0.25  # between two requests
+        self.cache_ttl = 60.0  # how long "is this channel connected?" is remembered
+        self.tell_every = 86400.0  # the same warning goes to the owners at most this often
+        self.max_ids = 300  # more changed ids than this in one channel: look at the whole channel instead
+        self.max_tries = 8  # a message that stays "too fresh" is given up after this many rounds
+        self._busy = asyncio.Lock()  # one look at a time
+        self._edits: dict = {}  # channel id -> message ids edited
+        self._news: dict = {}  # channel id -> message ids that are new posts
+        self._tries: dict = {}
+        self._stamp = 0
+        self._wake = asyncio.Event()
+        self._tasks: list = []
+        self._cache: dict = {}
+        self._told: dict = {}
+        self.last_reports: dict = {}  # channel id -> its latest SyncReport
+
+    # ----------------------------------------------------------------------------------------- running
+    @property
+    def enabled(self) -> bool:
+        return self.ctx.cfg.sync_interval_minutes > 0
+
+    def start(self) -> None:
+        if self._tasks or not self.enabled:
+            return
+        self._tasks = [
+            asyncio.create_task(self._periodic(), name="sync-timer"),
+            asyncio.create_task(self._events(), name="sync-live"),
+        ]
+
+    async def stop(self) -> None:
+        tasks, self._tasks = self._tasks, []
+        for t in tasks:
+            t.cancel()
+        for t in tasks:
+            try:
+                await t
+            except (asyncio.CancelledError, Exception):
+                pass
+
+    async def _periodic(self) -> None:
+        await asyncio.sleep(self.start_delay)
+        while True:
+            try:
+                await self.run_all()
+            except asyncio.CancelledError:
+                raise
+            except Exception:
+                log.exception("the timed check of the channels failed")
+            await asyncio.sleep(max(5, self.ctx.cfg.sync_interval_minutes) * 60)
+
+    async def _events(self) -> None:
+        while True:
+            await self._wake.wait()
+            self._wake.clear()
+            try:
+                await self.flush_events()
+            except asyncio.CancelledError:
+                raise
+            except Exception:
+                log.exception("working off the channel changes failed")
+
+    # ----------------------------------------------------------------------------------------- helpers
+    async def channel(self, cid: int):
+        """The connected channel with this id, or None (remembered for a minute: busy channels cost no database reads)."""
+        now = time.monotonic()
+        hit = self._cache.get(cid)
+        if hit is not None and now - hit[0] < self.cache_ttl:
+            return hit[1]
+        ch = await self.ctx.db.get_channel(cid)
+        if ch is not None and not ch.active:
+            ch = None
+        self._cache[cid] = (now, ch)
+        return ch
+
+    def forget_cache(self) -> None:
+        self._cache.clear()
+
+    def options(self, **kw) -> SyncOptions:
+        ctx = self.ctx
+        base = dict(delete_services=ctx.cfg.delete_service_messages, ignore=ctx.sync_ignore, pause=self.pause)
+        base.update(kw)
+        return SyncOptions(**base)
+
+    async def tell_owners(self, ch, key: str, text: str) -> None:
+        """A warning for the owners - the same one (per channel and kind) at most once a day."""
+        now = time.monotonic()
+        k = (ch.id, key)
+        if now - self._told.get(k, -1e12) < self.tell_every:
+            return
+        self._told[k] = now
+        for uid in sorted(self.ctx.cfg.owners):
+            try:
+                await self.ctx.client.send_message(uid, text, link_preview=False)
+            except Exception:
+                log.warning("could not warn user %s about %s", uid, ch.id, exc_info=True)
+
+    async def _after(self, ch, rep: SyncReport) -> None:
+        self.last_reports[ch.id] = rep
+        if rep.changed or rep.error or rep.anomaly:
+            log.info(
+                "channel %s: %s removed, %s updated, %s new, %s service messages deleted%s",
+                ch.id, len(rep.deleted), len(rep.edited), len(rep.adopted), rep.services_deleted,
+                f" - {rep.error or rep.anomaly}" if (rep.error or rep.anomaly) else "",
+            )
+        if rep.services_failed and rep.services_error in RIGHTS_ERRORS:
+            await self.tell_owners(ch, "rights", self._rights_text(ch, rep.services_error))
+
+    @staticmethod
+    def _rights_text(ch, why: Optional[str]) -> str:
+        return (
+            f"⚠️ I can't delete the service messages (name changes, “pinned a message” ...) in <b>{esc(ch.title)}</b> "
+            f"({esc(why or 'refused')}). Give the bot the <i>Delete messages</i> right there, or set "
+            "DELETE_SERVICE_MESSAGES=false to stop trying."
+        )
+
+    # ------------------------------------------------------------------------------------ full looks
+    async def run_all(self, chans: Optional[list] = None, *, explicit: bool = False, clean: bool = False, progress=None) -> list:
+        """Look at the connected channels one after the other. `explicit` (the /sync command): the caller holds the
+        one-long-job lock and everything is read; the timed check gives way to any long job instead."""
+        ctx = self.ctx
+        if chans is None:
+            chans = await ctx.db.list_channels()
+        busy = (lambda: False) if explicit else ctx.lock.locked
+        reports: list = []
+        for ch in chans:
+            if busy():
+                break
+            fallback = deleter = None
+            if clean and ctx.userbot is not None:
+                fallback, deleter = await ctx.userbot.fallback_for(ch)
+
+            async def prog(rep, ch=ch):
+                if progress is not None:
+                    await progress(ch, rep)
+
+            try:
+                async with self._busy:
+                    opts = self.options(deep=explicit, clean=clean, fallback=fallback)
+                    rep = await sync_channel(ctx.client, ctx.db, ch, opts, busy=busy, progress=prog)
+            except asyncio.CancelledError:
+                raise
+            except Exception as e:
+                log.exception("looking at channel %s failed", ch.id)
+                rep = SyncReport(channel=ch, error=f"{type(e).__name__}: {str(e)[:120]}")
+            finally:
+                if deleter is not None:
+                    await deleter.close()
+            await self._after(ch, rep)
+            reports.append(rep)
+        return reports
+
+    # ----------------------------------------------------------------------------- live: what updates say
+    def _touch(self) -> None:
+        self._stamp += 1
+        self._wake.set()
+
+    def note_new(self, cid: int, msg) -> None:
+        """A post appeared in a channel. The bot's own posts are skipped (it saves them itself)."""
+        if not self.enabled or self.ctx.lock.locked() or getattr(msg, "out", False):
+            return
+        self._news.setdefault(cid, set()).add(msg.id)
+        self._touch()
+
+    def note_edit(self, cid: int, msg) -> None:
+        if not self.enabled or self.ctx.lock.locked():
+            return
+        self._edits.setdefault(cid, set()).add(msg.id)
+        self._touch()
+
+    async def note_deleted(self, cid: int, ids) -> None:
+        """Posts were deleted in a channel: they leave My posts at once (nothing to read from Telegram for that)."""
+        if not self.enabled:
+            return
+        ch = await self.channel(cid)
+        if ch is None:
+            return
+        for pending in (self._edits, self._news):
+            if cid in pending:
+                pending[cid].difference_update(ids)
+        n = await self.ctx.db.forget_posts(cid, list(ids))
+        if n:
+            log.info("%s post(s) were deleted in channel %s: removed from My posts", n, cid)
+
+    async def on_service(self, cid: int, msg) -> None:
+        """A service message appeared: delete it."""
+        if not self.ctx.cfg.delete_service_messages:
+            return
+        ch = await self.channel(cid)
+        if ch is None:
+            return
+        out = await remove(self.ctx.client, peer_of(ch), [msg.id])
+        if out.error is None:
+            log.info("removed the “%s” notice %s from channel %s", action_label(msg), msg.id, cid)
+            return
+        log.warning("could not delete the “%s” notice %s in channel %s: %s", action_label(msg), msg.id, cid, out.error_name)
+        if out.error_name in RIGHTS_ERRORS:
+            await self.tell_owners(ch, "rights", self._rights_text(ch, out.error_name))
+
+    def _pending(self) -> bool:
+        return bool(self._edits or self._news)
+
+    async def flush_events(self) -> None:
+        """Work off the queue once it has been quiet for a moment (and no long job is running)."""
+        first = time.monotonic()
+        while self._pending():
+            stamp = self._stamp
+            await asyncio.sleep(self.debounce)
+            if stamp != self._stamp and time.monotonic() - first < self.max_debounce:
+                continue  # more is still coming in: let it settle
+            if self.ctx.lock.locked():
+                await asyncio.sleep(self.lock_wait)
+                continue
+            edits, news = self._edits, self._news
+            self._edits, self._news = {}, {}
+            await self._process(edits, news)
+            first = time.monotonic()
+
+    def _requeue(self, pending: dict, cid: int, mid: int) -> None:
+        key = (cid, mid)
+        self._tries[key] = self._tries.get(key, 0) + 1
+        if self._tries[key] > self.max_tries:
+            self._tries.pop(key, None)
+            return
+        pending.setdefault(cid, set()).add(mid)
+
+    async def _process(self, edits: dict, news: dict) -> None:
+        ctx = self.ctx
+        for cid in sorted(set(edits) | set(news)):
+            ch = await self.channel(cid)
+            if ch is None:
+                continue
+            e_ids, n_ids = set(edits.get(cid, ())), set(news.get(cid, ()))
+            opts = self.options(grace=self.rt_grace)
+            try:
+                async with self._busy:
+                    if len(e_ids | n_ids) > self.max_ids:  # a burst: one look at the whole channel is cheaper
+                        rep = await sync_channel(ctx.client, ctx.db, ch, opts, busy=ctx.lock.locked)
+                    else:
+                        rep = await sync_ids(
+                            ctx.client, ctx.db, ch, e_ids | n_ids, opts, adopt_ids=n_ids, busy=ctx.lock.locked
+                        )
+            except asyncio.CancelledError:
+                raise
+            except Exception as e:
+                log.exception("looking at the changes in channel %s failed", cid)
+                rep = SyncReport(channel=ch, error=f"{type(e).__name__}: {str(e)[:120]}")
+            await self._after(ch, rep)
+            if rep.busy:  # a long job started in the meantime: let the next timed check have it
+                continue
+            for mid in set(rep.young):
+                self._requeue(self._edits, cid, mid)
+            for mid in set(rep.young_new):
+                self._requeue(self._news, cid, mid)
+            for mid in (e_ids | n_ids) - set(rep.young) - set(rep.young_new):
+                self._tries.pop((cid, mid), None)

app/tgutil.py

@@ -138,6 +138,13 @@ def with_url(b, new_url: str):
     return nb
 
 
+def with_text(b, new_text: str):
+    """Copy of button `b` with another name; the link, colour ... are kept."""
+    nb = copy.copy(b)
+    nb.text = new_text
+    return nb
+
+
 def _style_dump(b) -> Optional[str]:
     st = getattr(b, "style", None)
     if st is None:

app/ui.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 
 from telethon import Button
 
-from .buttons import count, grid_label, missing_links
+from .buttons import count, grid_label, link_positions, missing_links
 from .tgutil import esc, post_link, short
 
 CANCEL = [[Button.inline("✖️ Cancel", "cx")]]
@@ -34,7 +34,7 @@ def panel_text(p, ch, note=None) -> str:
     n, miss = count(p.buttons), missing_links(p.buttons)
     lines.append(f"Buttons: {n}" + (f" (⚠️ {miss} without a link)" if miss else ""))
     if p.source == "adopted":
-        lines.append("<i>Picked up from the channel by /replace.</i>")
+        lines.append("<i>Picked up from the channel - it was not made with this bot.</i>")
     snippet = short(p.text or "", 220) or "(no text)"
     lines += ["", f"<i>{esc(snippet)}</i>"]
     if note:
@@ -64,6 +64,8 @@ def panel_keyboard(p) -> list:
         extra.append(Button.inline("🚫 Remove media", f"rm:{pid}"))
     if extra:
         kb.append(extra)
+    if p.status == "sent" and p.message_id and link_positions(p.buttons):
+        kb.append([Button.inline("🔗 Mass replace links", f"mlp:{pid}")])
     if p.status == "draft":
         kb.append([Button.inline("👁 Preview", f"pv:{pid}"), Button.inline("🚀 Publish", f"pb:{pid}")])
         kb.append([Button.inline("🗑 Discard", f"dl:{pid}"), Button.inline("📚 My posts", "pc:d:0")])

bot.py

@@ -13,8 +13,23 @@ from app.common import Ctx
 from app.config import Config, load_config
 from app.db import Database
 from app.errorlog import ErrorReporter, TelegramLogHandler
-from app.handlers import basic, channels, checkbtn, create, posts, replace, repost
+from app.expiry import Expirer
+from app.handlers import (
+    autopost,
+    basic,
+    broadcast,
+    channels,
+    checkbtn,
+    create,
+    masslinks,
+    posts,
+    replace,
+    repost,
+    shift,
+    sync,
+)
 from app.health import start_health_server
+from app.syncer import Syncer
 from app.userbot import Userbot
 
 log = logging.getLogger("bot")
@@ -22,13 +37,18 @@ log = logging.getLogger("bot")
 COMMANDS = [
     ("new", "Create a post"),
     ("posts", "My posts and drafts"),
+    ("autopost", "Post a series of episode download buttons"),
     ("addchannel", "Register a channel"),
     ("channels", "List channels"),
+    ("broadcast", "Send one message to all channels, optionally timed (owner)"),
     ("replace", "Swap a username in links (owner)"),
+    ("handleswap", "Swap a @username written in posts (owner)"),
     ("repost", "Copy a whole channel in order (owner)"),
+    ("fixlinks", "Point links at reposted posts (owner)"),
+    ("shift", "Copy posts from one channel to another (owner)"),
+    ("sync", "Compare channels with My posts now (owner)"),
     ("undo", "Undo the last replace (owner)"),
     ("userbot", "Status of the helper account that deletes old posts (owner)"),
-    ("testerror", "Send a test error to the log chat (owner)"),
     ("export", "Backup as JSON (owner)"),
     ("cancel", "Cancel what I'm doing"),
     ("help", "Show help"),
@@ -47,7 +67,7 @@ def build_client(cfg: Config) -> TelegramClient:
 
 
 def register_all(ctx: Ctx) -> None:
-    for module in (basic, channels, create, posts, checkbtn, replace, repost):
+    for module in (basic, channels, create, posts, masslinks, autopost, broadcast, checkbtn, replace, repost, shift, sync):
         module.register(ctx)
 
 
@@ -88,11 +108,14 @@ async def main() -> None:
     asyncio.get_running_loop().set_exception_handler(reporter.loop_exception_handler)
 
     db = None
+    ctx = None
     userbot = Userbot(client, cfg)
     try:
         db = Database(cfg.database_url, cfg.db_pool)
         await db.init()
         ctx = Ctx(cfg=cfg, db=db, client=client, userbot=userbot, reporter=reporter)
+        ctx.expirer = Expirer(ctx)
+        ctx.syncer = Syncer(ctx)
         register_all(ctx)
         await client.start(bot_token=cfg.bot_token)
         me = await client.get_me()
@@ -110,11 +133,17 @@ async def main() -> None:
             log.info("error log chat: off")
         if userbot.enabled:
             await userbot.connect()  # never fatal: the userbot is only needed for old posts
+        ctx.expirer.start()  # deletes timed broadcasts when their time is up
+        ctx.syncer.start()  # compares the channels with My posts now and then (and notices changes made meanwhile)
         await client.run_until_disconnected()
     except Exception as e:
         await reporter.report_fatal("The bot stopped because of an unhandled exception", e)
         raise
     finally:
+        if ctx is not None and ctx.syncer is not None:
+            await ctx.syncer.stop()
+        if ctx is not None and ctx.expirer is not None:
+            await ctx.expirer.stop()
         await userbot.close()
         if db is not None:
             await db.close()

tests/fakes.py

@@ -8,7 +8,7 @@ from telethon.tl.custom import Message
 from app.tgutil import make_url_button
 
 
-def make_msg(i, text="", entities=None, markup=None, out=False, media=None, grouped_id=None):
+def make_msg(i, text="", entities=None, markup=None, out=False, media=None, grouped_id=None, reply_to=None, pinned=False):
     return Message(
         id=i,
         peer_id=types.PeerChannel(1),
@@ -19,6 +19,8 @@ def make_msg(i, text="", entities=None, markup=None, out=False, media=None, grou
         reply_markup=markup,
         media=media,
         grouped_id=grouped_id,
+        reply_to=types.MessageReplyHeader(reply_to_msg_id=reply_to) if reply_to else None,
+        pinned=True if pinned else None,
     )
 
 
@@ -102,13 +104,18 @@ def _empty_rights(r):
 class FakeChannelClient:
     """Telegram's side of ONE channel (peer id 1) as seen by the bot, with switches for the odd things Telegram does."""
 
-    def __init__(self, msgs=None, *, username=None):
+    def __init__(self, msgs=None, *, username=None, cid=1, title="Test"):
+        self.cid = cid  # this simulation is the channel with this id; add_channel() puts more channels next to it
+        self.title = title
+        self.registry = {cid: self}
+        self.invite_hash = None  # a t.me/+hash invite link of this channel the bot "is in" (for CheckChatInvite)
         self.msgs = dict(msgs or {})
         self.username = username
         self.requests = []  # every request received, in order
         self.edits = []  # EditMessageRequests Telegram accepted
         self.delete_requests = []  # the id lists of channels.deleteMessages calls
         self._top = max(self.msgs, default=0)
+        self._pts = self._top  # the channel's event counter: new messages, edits, deletes and pins all move it
         self._gid = 7000
         self._media = {}
         for m in self.msgs.values():
@@ -131,9 +138,24 @@ class FakeChannelClient:
         self.require_approval = False
         self.users = {}  # user id -> types.User, for participant lists
         self.cant_delete_at_all = set()  # ids nobody can delete (userbot included)
+        self.pinned_order = []  # message ids in the order they were pinned
         self._n_invites = 0
 
     # ----------------------------------------------------------------------------------- helpers
+    def add_channel(self, other):
+        """Put another channel next to this one: requests that name it (by channel id) are answered by it."""
+        other.registry = self.registry
+        self.registry[other.cid] = other
+        self._media.update(other._media)
+        other._media = self._media  # a photo or file is known to the account, whichever channel it is sent to
+        return other
+
+    def _target(self, req):
+        """The channel a request is about (a forward is carried out by its destination)."""
+        peer = getattr(req, "to_peer", None) or getattr(req, "peer", None) or getattr(req, "channel", None)
+        cid = getattr(peer, "channel_id", None)
+        return self.registry.get(cid, self) if cid is not None else self
+
     def _remember_media(self, m):
         md = getattr(m, "media", None)
         if isinstance(md, types.MessageMediaPhoto) and md.photo:
@@ -162,11 +184,12 @@ class FakeChannelClient:
         else:
             m._hidden, m.reply_markup = None, markup
 
-    def _new(self, text, entities, markup, media=None, gid=None, invert=None, hide=False):
+    def _new(self, text, entities, markup, media=None, gid=None, invert=None, hide=False, reply_to=None):
         self._top += 1
+        self._pts += 1
         m = Message(
             id=self._top,
-            peer_id=types.PeerChannel(1),
+            peer_id=types.PeerChannel(self.cid),
             date=NOW,
             message=text,
             out=True,
@@ -174,6 +197,7 @@ class FakeChannelClient:
             media=media,
             grouped_id=gid,
             invert_media=invert,
+            reply_to=types.MessageReplyHeader(reply_to_msg_id=reply_to) if reply_to else None,
         )
         if self.hide_keyboard_on_create > 0 and markup is not None:
             hide = True
@@ -194,11 +218,15 @@ class FakeChannelClient:
 
     def _channel_obj(self):
         return types.Channel(
-            id=1, title="Test", photo=types.ChatPhotoEmpty(), date=NOW, access_hash=USER_HASH, username=self.username, broadcast=True
+            id=self.cid, title=self.title, photo=types.ChatPhotoEmpty(), date=NOW, access_hash=USER_HASH,
+            username=self.username, broadcast=True,
         )
 
     # --------------------------------------------------------------------------------- dispatcher
     async def __call__(self, req):
+        target = self._target(req)
+        if target is not self:
+            return await target(req)
         bytes(req)  # every request the bot builds must serialise
         self.requests.append(req)
         name = type(req).__name__
@@ -211,7 +239,11 @@ class FakeChannelClient:
         return handler(req)
 
     async def get_messages(self, peer, ids=None):
-        return [self.msgs.get(i) for i in ids]
+        chan = self.registry.get(getattr(peer, "channel_id", None), self)
+        err = chan.fail.get("GetMessagesRequest")  # reading a channel the bot may not read (a real client sends this request)
+        if err is not None:
+            raise err
+        return [chan.msgs.get(i) for i in ids]
 
     async def get_input_entity(self, peer):
         if isinstance(peer, int) and peer in self.known_users:
@@ -227,7 +259,7 @@ class FakeChannelClient:
         for ref in req.id:
             m = self.msgs.get(ref.id)
             if m is None:
-                out.append(types.MessageEmpty(id=ref.id, peer_id=types.PeerChannel(1)))
+                out.append(types.MessageEmpty(id=ref.id, peer_id=types.PeerChannel(self.cid)))
             elif isinstance(m, (types.MessageService, types.MessageEmpty)):
                 out.append(m)
             else:
@@ -235,7 +267,7 @@ class FakeChannelClient:
                     types.Message(
                         id=m.id, peer_id=m.peer_id, date=m.date, message=m.message, out=m.out, entities=m.entities,
                         reply_markup=m.reply_markup, media=m.media, grouped_id=m.grouped_id,
-                        invert_media=getattr(m, "invert_media", None),
+                        invert_media=getattr(m, "invert_media", None), reply_to=m.reply_to, pinned=m.pinned,
                     )
                 )
         return types.messages.ChannelMessages(
@@ -243,11 +275,12 @@ class FakeChannelClient:
         )
 
     def _do_ForwardMessagesRequest(self, req):
-        if self.restrict_forwards:
+        origin = self.registry.get(getattr(req.from_peer, "channel_id", None), self)
+        if origin.restrict_forwards:  # "Restrict saving content" is a setting of the channel the posts come from
             raise rpc("ChatForwardsRestrictedError")
         out, gids = [], {}
         for i in req.id:
-            src = self.msgs.get(i)
+            src = origin.msgs.get(i)
             if src is None:
                 continue
             gid = gids.setdefault(src.grouped_id, self._next_gid()) if src.grouped_id else None
@@ -265,18 +298,58 @@ class FakeChannelClient:
         self._gid += 1
         return self._gid
 
+    @staticmethod
+    def _reply_id(req):
+        r = getattr(req, "reply_to", None)
+        return r.reply_to_msg_id if r is not None else None
+
+    def _check_reply(self, req):
+        rid = self._reply_id(req)
+        if rid is not None and rid not in self.msgs:
+            raise rpc("MessageIdInvalidError")  # REPLY_MESSAGE_ID_INVALID
+        return rid
+
     def _do_SendMessageRequest(self, req):
-        return self._updates([self._new(req.message, req.entities, req.reply_markup, None, None, req.invert_media)])
+        rid = self._check_reply(req)
+        return self._updates([self._new(req.message, req.entities, req.reply_markup, None, None, req.invert_media, reply_to=rid)])
 
     def _do_SendMediaRequest(self, req):
         media = self._media_for(req.media)
-        return self._updates([self._new(req.message, req.entities, req.reply_markup, media, None, req.invert_media)])
+        rid = self._check_reply(req)
+        return self._updates([self._new(req.message, req.entities, req.reply_markup, media, None, req.invert_media, reply_to=rid)])
 
     def _do_SendMultiMediaRequest(self, req):
         gid = self._next_gid()
-        out = [self._new(s.message, s.entities, None, self._media_for(s.media), gid, req.invert_media) for s in req.multi_media]
+        rid = self._check_reply(req)
+        out = [
+            self._new(s.message, s.entities, None, self._media_for(s.media), gid, req.invert_media, reply_to=rid)
+            for s in req.multi_media
+        ]
         return self._updates(out)
 
+    def _do_UpdatePinnedMessageRequest(self, req):
+        """Pinning in a channel needs the "Edit messages of others" right; Telegram adds a "pinned" notice to the channel."""
+        if not self.rights.get("edit"):
+            raise rpc("ChatAdminRequiredError")
+        m = self.msgs.get(req.id)
+        if m is None or isinstance(m, types.MessageService):
+            raise rpc("MessageIdInvalidError")
+        if req.unpin:
+            m.pinned = None
+            return self._updates([])
+        if m.pinned:
+            raise rpc("MessageNotModifiedError")
+        m.pinned = True
+        self.pinned_order.append(req.id)
+        self._top += 1
+        self._pts += 2
+        note = types.MessageService(
+            id=self._top, peer_id=types.PeerChannel(self.cid), date=NOW, action=types.MessageActionPinMessage(),
+            reply_to=types.MessageReplyHeader(reply_to_msg_id=req.id),
+        )
+        self.msgs[note.id] = note
+        return self._updates([note])
+
     def _do_EditMessageRequest(self, req):
         m = self.msgs.get(req.id)
         if m is None:
@@ -295,6 +368,7 @@ class FakeChannelClient:
         m.message, m.entities = text, (ents or None)
         self._set_markup(m, markup)
         self.edits.append(req)
+        self._pts += 1
         return self._updates([])
 
     def _do_DeleteMessagesRequest(self, req):
@@ -308,12 +382,13 @@ class FakeChannelClient:
                 continue
             del self.msgs[i]
             n += 1
-        return types.messages.AffectedMessages(pts=self._top, pts_count=n)
+        self._pts += n
+        return types.messages.AffectedMessages(pts=self._pts, pts_count=n)
 
     # ------------------------------------------------------------------------ channel / rights
     def _do_GetFullChannelRequest(self, req):
         inv = types.ChatInviteExported(link=self.primary_invite, admin_id=1, date=NOW) if self.primary_invite else None
-        return SimpleNamespace(full_chat=SimpleNamespace(pts=self._top, exported_invite=inv))
+        return SimpleNamespace(full_chat=SimpleNamespace(pts=max(self._pts, self._top), exported_invite=inv))
 
     def _do_GetParticipantRequest(self, req):
         r = self.rights

tests/harness.py

@@ -117,9 +117,29 @@ class FakeTG(FakeChannelClient):
         self.deleted.append(list(ids))
         return [SimpleNamespace(pts_count=len(ids))]
 
+    async def get_entity(self, what):
+        """Public channels by @username, or any channel by PeerChannel - the ones put next to this one."""
+        for chan in self.registry.values():
+            if isinstance(what, str) and chan.username and chan.username.lower() == what.lstrip("@").lower():
+                return chan._channel_obj()
+            if isinstance(what, types.PeerChannel) and what.channel_id == chan.cid:
+                return chan._channel_obj()
+        raise ValueError(f"Cannot find any entity corresponding to {what!r}")
+
     async def __call__(self, req):
+        if isinstance(req, functions.messages.CheckChatInviteRequest):
+            for chan in self.registry.values():
+                if chan.invite_hash == req.hash:
+                    return types.ChatInviteAlready(chat=chan._channel_obj())
+            raise ValueError("INVITE_HASH_INVALID")
+        target = self._target(req)
+        if target is not self:
+            return await target(req)
         if isinstance(req, functions.messages.EditMessageRequest):
             self.edits.append(("raw", req.id, req))
+            err = self.fail.get("EditMessageRequest")
+            if err is not None:
+                raise err
             if req.id not in self.msgs:  # a post the simulation doesn't hold: just note the edit
                 return None
         if isinstance(req, functions.channels.DeleteMessagesRequest):
@@ -163,6 +183,12 @@ class App:
             await cb(ev)
         return ev
 
+    async def raw(self, update):
+        """Hand a raw Telegram update (a new / edited / deleted channel message ...) to the handlers that listen for it."""
+        for builder, cb in self.tg.handlers:
+            if isinstance(builder, events.Raw) and builder.filter(update) is not None:
+                await cb(update)
+
     async def press(self, data):
         ev = FakeEvent(self.uid, self.out, data=data)
         for builder, cb in self.tg.handlers:

tests/test_autopost.py

@@ -0,0 +1,495 @@
+"""/autopost (a series of episode posts, then the real links) and "Mass replace links" in a post's panel."""
+import asyncio
+
+import pytest
+from telethon import errors
+
+from app.autopost import (
+    PLACEHOLDER_LINK,
+    button_label,
+    parse_autopost_args,
+    plan_ranges,
+    post_text,
+    ranges_summary,
+)
+from app.tgutil import button_url
+
+from .harness import App, norm
+
+
+def run(coro):
+    return asyncio.run(coro)
+
+
+async def make_app(tmp_path, **cfg):
+    app = App(tmp_path, edit_delay=0, **cfg)
+    await app.start()
+    await app.db.save_channel(1, 5, "Anime Channel", "animech", 1)
+    return app
+
+
+def sent_markup(entry):
+    return entry[3]["buttons"]
+
+
+def link_edits(app):
+    """(message id, new link) for every keyboard-only edit the bot made."""
+    out = []
+    for kind, mid, req in app.tg.edits:
+        if kind == "raw" and req.message is None and req.reply_markup is not None:
+            out.append((mid, button_url(req.reply_markup.rows[0].buttons[0])))
+    return out
+
+
+# ------------------------------------------------------------------------------- the plan
+def test_the_ranges_follow_the_interval():
+    assert plan_ranges(114, 20) == [(1, 20), (21, 40), (41, 60), (61, 80), (81, 100), (101, 114)]
+    assert plan_ranges(100, 20) == [(1, 20), (21, 40), (41, 60), (61, 80), (81, 100)]
+    assert plan_ranges(20, 20) == [(1, 20)]
+
+
+def test_a_short_end_is_added_to_the_post_before_it():
+    assert plan_ranges(102, 20)[-2:] == [(61, 80), (81, 102)]  # 2 left over: less than half of 20
+    assert plan_ranges(110, 20)[-1] == (81, 110)  # exactly half of 20: still added
+    assert plan_ranges(111, 20)[-2:] == [(81, 100), (101, 111)]  # 11 is more than half: its own post
+    assert plan_ranges(114, 20)[-1] == (101, 114)  # 14 is more than half
+    assert plan_ranges(22, 15) == [(1, 22)]  # half of 15 is 7.5: 7 left over is added ...
+    assert plan_ranges(23, 15) == [(1, 15), (16, 23)]  # ... 8 is not
+
+
+def test_a_total_below_the_interval_is_one_post_and_small_intervals_work():
+    assert plan_ranges(14, 20) == [(1, 14)]
+    assert plan_ranges(3, 20) == [(1, 3)]
+    assert plan_ranges(3, 1) == [(1, 1), (2, 2), (3, 3)]
+    assert plan_ranges(5, 2) == [(1, 2), (3, 5)]  # one left over = half of 2: added to the post before it
+    with pytest.raises(ValueError):
+        plan_ranges(0, 20)
+
+
+def test_every_episode_is_in_exactly_one_post():
+    for total in range(1, 130):
+        for interval in (1, 2, 3, 7, 10, 15, 20, 25):
+            ranges = plan_ranges(total, interval)
+            flat = [n for a, b in ranges for n in range(a, b + 1)]
+            assert flat == list(range(1, total + 1)), (total, interval)
+            assert all(b - a + 1 <= interval + interval // 2 for a, b in ranges)
+
+
+def test_texts_are_the_ones_asked_for():
+    assert post_text(1, 20) == "Episodes 01 to 20\n\nClick the button below to download 👇"
+    assert post_text(101, 114) == "Episodes 101 to 114\n\nClick the button below to download 👇"
+    assert button_label(21, 40) == "Download Episodes 21 to 40"
+    assert button_label(5, 5) == "Download Episode 05"
+    assert ranges_summary(plan_ranges(114, 20)) == "01–20 · 21–40 · 41–60 · 61–80 · 81–100 · 101–114"
+    assert ranges_summary([(i, i) for i in range(1, 40)]).endswith("… · 38 · 39")
+
+
+def test_the_arguments():
+    assert parse_autopost_args("114 20") == (114, 20)
+    assert parse_autopost_args("  12,  5 ") == (12, 5)
+    for bad in ("", "114", "114 20 3", "a b", "0 20", "20 0", "-5 3", "1.5 2", "999999999 20"):
+        with pytest.raises(ValueError):
+            parse_autopost_args(bad)
+
+
+# ------------------------------------------------------------------------------- the command
+def test_the_whole_autopost_flow(tmp_path):
+    async def go():
+        app = await make_app(tmp_path)
+        await app.text("/autopost 114 20")
+        shown = norm(app.out.last_text)
+        assert "6 post(s)" in shown and "01–20 · 21–40 · 41–60 · 61–80 · 81–100 · 101–114" in shown
+        assert "Which channel" in shown and "Anime Channel" in str(app.out.last_buttons())
+
+        await app.press(app.out.callback_data("Anime Channel"))
+        posted = [e for e in app.tg.sent if e[0] == "text" and e[1].channel_id == 1]
+        assert len(posted) == 6
+        assert [e[2] for e in posted] == [post_text(a, b) for a, b in plan_ranges(114, 20)]
+        for (a, b), entry in zip(plan_ranges(114, 20), posted):
+            row = sent_markup(entry).rows[0].buttons
+            assert len(row) == 1 and row[0].text == f"Download Episodes {a:02d} to {b:02d}"
+            assert button_url(row[0]) == PLACEHOLDER_LINK == "https://t.me/English"
+            assert entry[3]["link_preview"] is False
+
+        # all six are in My posts, as published posts of this channel
+        posts = await app.db.sent_posts(1)
+        assert [p.message_id for p in posts] == [100, 101, 102, 103, 104, 105]
+        assert posts[0].source == "bot" and posts[0].buttons == [[{"t": "Download Episodes 01 to 20", "u": PLACEHOLDER_LINK}]]
+
+        # the first link is asked for at once, with a cancel button
+        ask = norm(app.out.last_text)
+        assert "1 of 6" in ask and "Download Episodes 01 to 20" in ask and PLACEHOLDER_LINK in ask
+        assert "Cancel" in str(app.out.last_buttons()) and "Skip" in str(app.out.last_buttons())
+
+        links = [f"https://t.me/mybot?start=part{i}" for i in range(1, 7)]
+        for i, url in enumerate(links):
+            await app.text(url)
+            if i < 5:
+                nxt = norm(app.out.last_text)
+                assert f"{i + 2} of 6" in nxt and "now opens" in nxt and url in nxt
+                a, b = plan_ranges(114, 20)[i + 1]
+                assert f"Download Episodes {a:02d} to {b:02d}" in nxt
+        assert "All done" in app.out.last_text and "6 link(s) replaced" in app.out.last_text
+
+        # each answer changed only the keyboard of its own message, at once
+        assert link_edits(app) == [(100 + i, links[i]) for i in range(6)]
+        assert [button_url(app.tg.edits[i][2].reply_markup.rows[0].buttons[0]) for i in range(6)] == links
+        posts = await app.db.sent_posts(1)
+        assert [p.buttons[0][0]["u"] for p in posts] == links
+        assert [p.buttons[0][0]["t"] for p in posts][-1] == "Download Episodes 101 to 114"
+        assert 1 not in app.ctx.state  # nothing left to answer
+        await app.db.close()
+
+    run(go())
+
+
+def test_the_short_end_gets_its_own_post_or_joins_the_last(tmp_path):
+    async def go():
+        app = await make_app(tmp_path)
+        await app.text("/autopost 102 20")
+        await app.press(app.out.callback_data("Anime Channel"))
+        texts = [e[2].splitlines()[0] for e in app.tg.sent if e[0] == "text" and e[1].channel_id == 1]
+        assert texts == ["Episodes 01 to 20", "Episodes 21 to 40", "Episodes 41 to 60", "Episodes 61 to 80", "Episodes 81 to 102"]
+        await app.db.close()
+
+    run(go())
+
+
+def test_admins_may_use_it_and_strangers_may_not(tmp_path):
+    async def go():
+        app = await make_app(tmp_path, admins=frozenset({7}))
+        app.uid = 7
+        await app.text("/autopost 40 20")
+        assert "Which channel" in norm(app.out.last_text)
+        await app.press(app.out.callback_data("Anime Channel"))
+        assert len([e for e in app.tg.sent if e[0] == "text" and e[1].channel_id == 1]) == 2
+        app.uid = 99
+        before = len(app.tg.sent)
+        await app.text("/autopost 40 20")
+        assert "private" in app.out.last_text and len(app.tg.sent) == before
+        await app.db.close()
+
+    run(go())
+
+
+def test_wrong_numbers_and_no_channels_are_explained(tmp_path):
+    async def go():
+        app = await make_app(tmp_path)
+        await app.text("/autopost")
+        assert "Usage" in norm(app.out.last_text) and "/autopost 114 20" in norm(app.out.last_text)
+        for bad in ("114", "x y", "0 5", "114 20 5"):
+            await app.text(f"/autopost {bad}")
+            assert "Usage: /autopost <total episodes> <interval>" in norm(app.out.last_text), bad
+        await app.text("/autopost 5000 1")
+        assert "at most 200" in app.out.last_text
+        assert not [e for e in app.tg.sent if e[0] == "text"]  # nothing was posted by any of those
+
+        (tmp_path / "empty").mkdir()
+        empty = App(tmp_path / "empty", edit_delay=0)
+        await empty.start()
+        await empty.text("/autopost 114 20")
+        assert "No channels yet" in empty.out.last_text
+        await empty.db.close()
+        await app.db.close()
+
+    run(go())
+
+
+def test_cancelling_the_channel_choice_posts_nothing(tmp_path):
+    async def go():
+        app = await make_app(tmp_path)
+        await app.text("/autopost 114 20")
+        await app.press(app.out.callback_data("Cancel"))
+        assert "Cancelled" in app.out.last_text
+        assert not [e for e in app.tg.sent if e[0] == "text"]
+        # an old button does nothing any more
+        await app.press("apc:" + "0" * 8 + ":1")
+        assert "expired" in app.out.last_text
+        await app.db.close()
+
+    run(go())
+
+
+def test_a_double_press_posts_once(tmp_path):
+    async def go():
+        app = await make_app(tmp_path)
+        await app.text("/autopost 40 20")
+        data = app.out.callback_data("Anime Channel")
+        await app.press(data)
+        await app.press(data)
+        assert "expired" in app.out.last_text
+        assert len([e for e in app.tg.sent if e[0] == "text" and e[1].channel_id == 1]) == 2
+        await app.db.close()
+
+    run(go())
+
+
+def test_the_bot_needs_the_post_right_and_a_free_hand(tmp_path):
+    async def go():
+        app = await make_app(tmp_path)
+        app.tg.rights = dict(admin=True, post=False, edit=True, delete=True, invite=False, add_admins=False)
+        await app.text("/autopost 40 20")
+        pick = app.out.callback_data("Anime Channel")
+        await app.press(pick)
+        assert "Post messages" in app.out.last_text
+        assert not [e for e in app.tg.sent if e[0] == "text"]
+        # the plan is still there: with the right given, the same button works - but not while a long job runs
+        app.tg.rights["post"] = True
+        await app.ctx.lock.acquire()
+        await app.press(pick)
+        assert "still running" in app.out.last_text
+        app.ctx.lock.release()
+        await app.press(pick)
+        assert len([e for e in app.tg.sent if e[0] == "text" and e[1].channel_id == 1]) == 2
+        await app.db.close()
+
+    run(go())
+
+
+def test_when_telegram_stops_midway_the_posts_made_still_get_their_links(tmp_path):
+    async def go():
+        app = await make_app(tmp_path)
+        real = app.tg.send_message
+        calls = []
+
+        async def flaky(peer, message="", **kw):
+            calls.append(message)
+            if len(calls) == 3:
+                raise errors.ChatWriteForbiddenError(request=None)
+            return await real(peer, message, **kw)
+
+        app.tg.send_message = flaky
+        await app.text("/autopost 114 20")
+        await app.press(app.out.callback_data("Anime Channel"))
+        # the summary (an edit of the picker) tells where it stopped; the next message asks for the first link
+        summary = [norm(t) for k, t, _ in app.out.log if k == "edit" and t and "Stopped early" in t]
+        assert summary and "Posted 2 of 6" in summary[0] and "Episodes 41 to 60" in summary[0]
+        assert "1 of 2" in norm(app.out.last_text)
+        assert len(await app.db.sent_posts(1)) == 2
+        await app.text("https://t.me/mybot?start=1")
+        await app.text("https://t.me/mybot?start=2")
+        assert "All done" in app.out.last_text and "2 link(s) replaced" in app.out.last_text
+        await app.db.close()
+
+    run(go())
+
+
+def test_nothing_posted_means_a_plain_error(tmp_path):
+    async def go():
+        app = await make_app(tmp_path)
+
+        async def refuse(peer, message="", **kw):
+            raise errors.ChatWriteForbiddenError(request=None)
+
+        app.tg.send_message = refuse
+        await app.text("/autopost 40 20")
+        await app.press(app.out.callback_data("Anime Channel"))
+        assert "Nothing was posted" in app.out.last_text and "Episodes 01 to 20" in app.out.last_text
+        assert await app.db.sent_posts(1) == []
+        await app.db.close()
+
+    run(go())
+
+
+def test_skipping_and_stopping_while_the_links_are_asked(tmp_path):
+    async def go():
+        app = await make_app(tmp_path)
+        await app.text("/autopost 80 20")
+        await app.press(app.out.callback_data("Anime Channel"))
+        await app.text("https://t.me/mybot?start=1")
+        assert "2 of 4" in norm(app.out.last_text)
+        await app.press(app.out.callback_data("Skip"))  # keeps the placeholder
+        assert "Skipped" in app.out.last_text and "3 of 4" in norm(app.out.last_text)
+        cancel = app.out.callback_data("Cancel")
+        # the Skip button of the earlier message is stale now
+        await app.press("lqs:1")
+        assert "earlier step" in app.out.last_text
+        await app.press(cancel)
+        shown = norm(app.out.last_text)
+        assert "Stopped" in shown and "1 link(s) replaced" in shown and "1 skipped" in shown and "2 not asked" in shown
+        assert 1 not in app.ctx.state
+        posts = await app.db.sent_posts(1)
+        assert [p.buttons[0][0]["u"] for p in posts] == ["https://t.me/mybot?start=1", PLACEHOLDER_LINK, PLACEHOLDER_LINK, PLACEHOLDER_LINK]
+        # nothing more is expected from the user
+        await app.text("https://t.me/whatever")
+        assert "Use /new" in app.out.last_text
+        await app.db.close()
+
+    run(go())
+
+
+def test_something_that_is_not_a_link_is_asked_again_and_a_refusal_keeps_the_step(tmp_path):
+    async def go():
+        app = await make_app(tmp_path)
+        await app.text("/autopost 40 20")
+        await app.press(app.out.callback_data("Anime Channel"))
+        await app.text("hello there")
+        assert "doesn't look like a link" in norm(app.out.last_text)
+        # Telegram refuses the keyboard edit: the saved link is unchanged and the same post is asked again
+        app.tg.fail["EditMessageRequest"] = errors.ChatAdminRequiredError(request=None)
+        await app.text("t.me/mybot?start=1")
+        assert "NOT changed" in app.out.last_text
+        assert (await app.db.sent_posts(1))[0].buttons[0][0]["u"] == PLACEHOLDER_LINK
+        app.tg.fail.clear()
+        await app.text("t.me/mybot?start=1")
+        assert "now opens" in app.out.last_text and "https://t.me/mybot?start=1" in app.out.last_text
+        assert (await app.db.sent_posts(1))[0].buttons[0][0]["u"] == "https://t.me/mybot?start=1"
+        await app.db.close()
+
+    run(go())
+
+
+# ------------------------------------------------------------------- mass replace links
+def sent(app, pid_text, message_id, buttons, channel_id=1, text=None):
+    return app.db.create_post(
+        channel_id=channel_id, message_id=message_id, status="sent", source="bot", text=text or pid_text,
+        entities=[], buttons=buttons, link_preview=False,
+    )
+
+
+def one(label, url):
+    return [[{"t": label, "u": url}]]
+
+
+async def channel_with_posts(app):
+    """#1 text, #2 button, #3 text, #4 only another bot's button, #5 button, #6 two buttons, #7 button in ANOTHER channel."""
+    await app.db.save_channel(2, 6, "Other", "other", 1)
+    posts = {}
+    posts[1] = await sent(app, "just text", 10, [])
+    posts[2] = await sent(app, "Episodes 01 to 20", 11, one("Download Episodes 01 to 20", "https://t.me/old?start=a"))
+    posts[3] = await sent(app, "plain again", 12, [])
+    posts[4] = await sent(app, "reactions only", 13, [[{"t": "👍 3", "raw": "AAAA"}]])
+    posts[5] = await sent(app, "Episodes 21 to 40", 14, one("Download Episodes 21 to 40", "https://t.me/old?start=b"))
+    posts[6] = await sent(
+        app, "Choose", 15, [[{"t": "480p", "u": "https://t.me/old?start=c"}, {"t": "720p", "u": "https://t.me/old?start=d"}]],
+    )
+    posts[7] = await sent(app, "elsewhere", 5, one("Other channel", "https://t.me/old?start=z"), channel_id=2)
+    return posts
+
+
+def test_only_published_posts_with_a_link_button_get_the_option(tmp_path):
+    async def go():
+        app = await make_app(tmp_path)
+        posts = await channel_with_posts(app)
+        draft = await app.db.create_post(channel_id=1, status="draft", source="bot", text="d", entities=[], buttons=one("Get", "https://t.me/x"))
+        for key, expected in ((1, False), (2, True), (4, False), (5, True), (6, True)):
+            await app.press(f"po:{posts[key].id}")
+            assert ("Mass replace links" in str(app.out.last_buttons())) is expected, key
+        await app.press(f"po:{draft.id}")
+        assert "Mass replace links" not in str(app.out.last_buttons())
+        await app.press(f"mlp:{draft.id}")
+        assert "published posts only" in app.out.last_text
+        await app.press(f"mlp:{posts[1].id}")
+        assert "no button with a link" in app.out.last_text
+        await app.db.close()
+
+    run(go())
+
+
+def test_mass_replace_walks_through_the_button_posts_of_the_channel(tmp_path):
+    async def go():
+        app = await make_app(tmp_path)
+        posts = await channel_with_posts(app)
+        await app.press(f"po:{posts[2].id}")
+        await app.press(app.out.callback_data("Mass replace links"))
+        ask = norm(app.out.last_text)
+        assert "Mass replace links" in ask and "1 of 4" in ask and "Download Episodes 01 to 20" in ask
+        assert "https://t.me/old?start=a" in ask and f"Post #{posts[2].id}" in ask
+        assert "Cancel" in str(app.out.last_buttons())
+
+        await app.text("https://t.me/new?start=1")
+        ask = norm(app.out.last_text)  # the text posts (#3) and the reactions-only post (#4) are skipped
+        assert "2 of 4" in ask and "Download Episodes 21 to 40" in ask and f"Post #{posts[5].id}" in ask
+        assert "now opens https://t.me/new?start=1" in ask
+        await app.text("https://t.me/new?start=2")
+        ask = norm(app.out.last_text)
+        assert "3 of 4" in ask and "480p" in ask and "https://t.me/old?start=c" in ask
+        await app.text("https://t.me/new?start=3")
+        assert "4 of 4" in norm(app.out.last_text) and "720p" in norm(app.out.last_text)
+        await app.text("https://t.me/new?start=4")
+        assert "All done" in app.out.last_text and "4 link(s) replaced" in app.out.last_text
+
+        # keyboard-only edits: the text of the posts was never sent again
+        edits = [(mid, req) for kind, mid, req in app.tg.edits if kind == "raw"]
+        assert [mid for mid, _ in edits] == [11, 14, 15, 15]
+        assert all(req.message is None for _, req in edits)
+        last = edits[-1][1].reply_markup.rows[0].buttons
+        assert [button_url(b) for b in last] == ["https://t.me/new?start=3", "https://t.me/new?start=4"]
+        assert [b.text for b in last] == ["480p", "720p"]
+
+        fresh = {k: await app.db.get_post(p.id) for k, p in posts.items()}
+        assert fresh[2].buttons == one("Download Episodes 01 to 20", "https://t.me/new?start=1")
+        assert fresh[5].buttons[0][0]["u"] == "https://t.me/new?start=2"
+        assert [b["u"] for b in fresh[6].buttons[0]] == ["https://t.me/new?start=3", "https://t.me/new?start=4"]
+        assert fresh[4].buttons == [[{"t": "👍 3", "raw": "AAAA"}]]  # another bot's button: untouched
+        assert fresh[7].buttons[0][0]["u"] == "https://t.me/old?start=z"  # another channel: untouched
+        assert fresh[2].text == "Episodes 01 to 20"
+        await app.db.close()
+
+    run(go())
+
+
+def test_it_starts_at_the_chosen_post_and_goes_towards_the_newest(tmp_path):
+    async def go():
+        app = await make_app(tmp_path)
+        posts = await channel_with_posts(app)
+        await app.press(f"mlp:{posts[5].id}")  # #2 is older: not asked
+        assert "1 of 3" in norm(app.out.last_text) and "Download Episodes 21 to 40" in norm(app.out.last_text)
+        await app.press(app.out.callback_data("Skip"))
+        assert "2 of 3" in norm(app.out.last_text) and "480p" in norm(app.out.last_text)
+        await app.press(app.out.callback_data("Cancel"))
+        shown = norm(app.out.last_text)
+        assert "Stopped" in shown and "1 skipped" in shown and "2 not asked" in shown
+        assert (await app.db.get_post(posts[5].id)).buttons[0][0]["u"] == "https://t.me/old?start=b"
+        assert not [1 for kind, mid, req in app.tg.edits if kind == "raw"]
+        await app.db.close()
+
+    run(go())
+
+
+def test_a_post_that_disappears_meanwhile_is_passed_over(tmp_path):
+    async def go():
+        app = await make_app(tmp_path)
+        posts = await channel_with_posts(app)
+        await app.press(f"mlp:{posts[2].id}")
+        await app.db.delete_post(posts[5].id)  # forgotten in the bot while the list is open
+        await app.text("https://t.me/new?start=1")
+        assert "3 of 4" in norm(app.out.last_text) and "480p" in norm(app.out.last_text)  # #5 was passed over
+        await app.text("https://t.me/new?start=2")
+        await app.text("https://t.me/new?start=3")
+        assert "All done" in app.out.last_text
+        assert "3 link(s) replaced" in app.out.last_text and "1 button(s) no longer there" in app.out.last_text
+        await app.db.close()
+
+    run(go())
+
+
+def test_telegram_refusing_a_link_keeps_the_same_step(tmp_path):
+    async def go():
+        app = await make_app(tmp_path)
+        posts = await channel_with_posts(app)
+        await app.press(f"mlp:{posts[2].id}")
+        skip = app.out.callback_data("Skip")
+        app.tg.fail["EditMessageRequest"] = errors.MessageIdInvalidError(request=None)
+        await app.text("https://t.me/new?start=1")
+        assert "NOT changed" in app.out.last_text
+        assert (await app.db.get_post(posts[2].id)).buttons[0][0]["u"] == "https://t.me/old?start=a"
+        app.tg.fail.clear()
+        await app.press(skip)
+        assert "2 of 4" in norm(app.out.last_text)
+        await app.db.close()
+
+    run(go())
+
+
+def test_the_last_button_post_ends_the_list(tmp_path):
+    async def go():
+        app = await make_app(tmp_path)
+        p = await sent(app, "only one", 3, one("Get", "https://t.me/old"))
+        await app.press(f"mlp:{p.id}")
+        assert "1 of 1" in norm(app.out.last_text)
+        await app.text("https://t.me/new")
+        assert "All done" in app.out.last_text and "1 link(s) replaced" in app.out.last_text
+        await app.db.close()
+
+    run(go())

tests/test_broadcast.py

@@ -0,0 +1,472 @@
+"""/broadcast (and its spelling /boardcast): one message to every channel, optionally deleted again after a time."""
+import asyncio
+from datetime import timedelta
+from types import SimpleNamespace
+
+import pytest
+from telethon import errors, types
+
+from app.db import Channel, as_utc, utcnow
+from app.expiry import MAX_ATTEMPTS, Expirer, human, parse_duration
+from app.tgutil import button_url
+
+from .fakes import FakeChannelClient, photo_media
+from .harness import App, norm
+
+
+def run(coro):
+    return asyncio.run(coro)
+
+
+def posted(app):
+    """Messages the bot sent to channels (not to people)."""
+    return [e for e in app.tg.sent if isinstance(e[1], types.InputPeerChannel)]
+
+
+async def make_app(tmp_path, channels=2, **cfg):
+    """Channel 1 "Anime Channel" is the app's own fake; channel 2 "Movie Channel" sits next to it. Posts the bot sends
+    to a channel really appear in it, so deleting them really deletes."""
+    app = App(tmp_path, edit_delay=0, **cfg)
+    await app.start()
+    app.tg.username, app.tg.title = "animech", "Anime Channel"
+    await app.db.save_channel(1, 5, "Anime Channel", "animech", 1)
+    if channels > 1:
+        app.tg.add_channel(FakeChannelClient({}, username="moviech", cid=2, title="Movie Channel"))
+        await app.db.save_channel(2, 6, "Movie Channel", "moviech", 1)
+
+    async def send_message(peer, message="", **kw):
+        if not isinstance(peer, types.InputPeerChannel):  # a person (the preview)
+            app.tg.sent.append(("text", peer, message, kw))
+            return SimpleNamespace(id=next(app.tg.ids))
+        app.tg.sent.append(("text", peer, message, kw))
+        return app.tg.registry[peer.channel_id]._new(message, kw.get("formatting_entities"), kw.get("buttons"))
+
+    async def send_file(peer, file, **kw):
+        app.tg.sent.append(("file", peer, file, kw))
+        if not isinstance(peer, types.InputPeerChannel):
+            return SimpleNamespace(id=next(app.tg.ids))
+        return app.tg.registry[peer.channel_id]._new(kw.get("caption"), kw.get("formatting_entities"), kw.get("buttons"), media=file)
+
+    app.tg.send_message, app.tg.send_file = send_message, send_file
+    app.ctx.expirer = Expirer(app.ctx)
+    return app
+
+
+async def broadcast(app, command="/broadcast 1h", text="Hello everyone", buttons=None, message=None):
+    await app.text(command)
+    await app.text(text, message=message)
+    if buttons is None:
+        await app.press(app.out.callback_data("No buttons"))
+    else:
+        await app.text(buttons)
+    await app.press(app.out.callback_data("Send to all channels"))
+
+
+# ----------------------------------------------------------------------------------- the time
+def test_times_are_minutes_hours_days():
+    assert parse_duration("50m") == 3000
+    assert parse_duration("1h") == 3600
+    assert parse_duration("5d") == 5 * 86400
+    assert parse_duration("1d12h") == 129600
+    assert parse_duration("90 min") == 5400
+    assert parse_duration("2 Hours") == 7200
+    assert parse_duration("1w") == 604800
+    for bad in ("", "soon", "10", "h", "0m", "30s", "5x", "400d", "1h 5"):
+        with pytest.raises(ValueError):
+            parse_duration(bad)
+
+
+def test_times_are_spelled_out():
+    assert human(3000) == "50 minutes"
+    assert human(3600) == "1 hour"
+    assert human(5400) == "1 hour 30 minutes"
+    assert human(129600) == "1 day 12 hours"
+    assert human(5 * 86400) == "5 days"
+    assert human(90061) == "1 day 1 hour"
+    assert human(20) == "under a minute"
+
+
+# ----------------------------------------------------------------------------------- sending
+def test_a_timed_broadcast_goes_to_every_channel_and_is_remembered(tmp_path):
+    async def go():
+        app = await make_app(tmp_path)
+        before = utcnow()
+        await app.text("/broadcast 1h")
+        assert "2 channel(s)" in norm(app.out.last_text) and "1 hour" in norm(app.out.last_text)
+        await app.text("Hello everyone")
+        assert "Buttons?" in norm(app.out.last_text)
+        await app.press(app.out.callback_data("No buttons"))
+        shown = norm(app.out.last_text)
+        assert "Send it to 2 channel(s)" in shown and "Anime Channel" in shown and "Movie Channel" in shown
+        assert "Deleted again 1 hour after sending" in shown
+        # the preview went to the person, nothing to the channels yet
+        assert [e[2] for e in app.tg.sent if not isinstance(e[1], types.InputPeerChannel)] == ["Hello everyone"]
+        assert posted(app) == []
+
+        await app.press(app.out.callback_data("Send to all channels"))
+        report = norm(app.out.last_text)
+        assert "Broadcast sent to 2 of 2" in report and "✅ Anime Channel" in report and "✅ Movie Channel" in report
+        assert "deleted from these channels and from My posts in 1 hour" in report and "UTC" in report
+        assert [e[2] for e in posted(app)] == ["Hello everyone", "Hello everyone"]
+        assert app.tg.registry[1].msgs[1].message == "Hello everyone" and app.tg.registry[2].msgs[1].message == "Hello everyone"
+
+        for cid in (1, 2):  # in My posts, as published posts of the bot
+            (p,) = await app.db.sent_posts(cid)
+            assert (p.message_id, p.text, p.source, p.created_by) == (1, "Hello everyone", "bot", 1)
+        rows = await app.db.pending_deletes()
+        assert sorted((r.channel_id, r.message_id) for r in rows) == [(1, 1), (2, 1)]
+        for r in rows:
+            assert as_utc(r.delete_at) - before >= timedelta(hours=1) and as_utc(r.delete_at) - utcnow() <= timedelta(hours=1)
+        assert len({r.group_id for r in rows}) == 1
+        assert app.ctx.expirer._next is not None
+        await app.db.close()
+
+    run(go())
+
+
+def test_without_a_time_the_message_stays_and_the_other_spelling_works(tmp_path):
+    async def go():
+        app = await make_app(tmp_path)
+        await broadcast(app, "/boardcast")
+        report = norm(app.out.last_text)
+        assert "Broadcast sent to 2 of 2" in report and "stays in the channels" in report
+        assert await app.db.pending_deletes() == []
+        assert app.ctx.expirer._next is None
+        assert len(posted(app)) == 2
+        await app.db.close()
+
+    run(go())
+
+
+def test_buttons_and_formatting(tmp_path):
+    async def go():
+        app = await make_app(tmp_path)
+        bold = types.MessageEntityBold(0, 5)
+        msg = SimpleNamespace(raw_text="Hello all", entities=[bold], media=None, grouped_id=None, forward=None)
+        await app.text("/broadcast 2d")
+        await app.text("Hello all", message=msg)
+        await app.text("Join - https://t.me/animech | Site - https://example.com/x\nMore - t.me/moviech")
+        assert "Send it to 2 channel(s)" in norm(app.out.last_text) and "2 days" in norm(app.out.last_text)
+        await app.press(app.out.callback_data("Send to all channels"))
+        first = posted(app)[0]
+        assert first[2] == "Hello all"
+        assert [type(e).__name__ for e in first[3]["formatting_entities"]] == ["MessageEntityBold"]
+        rows = first[3]["buttons"].rows
+        assert [[b.text for b in r.buttons] for r in rows] == [["Join", "Site"], ["More"]]
+        assert button_url(rows[1].buttons[0]) == "https://t.me/moviech"
+        (p,) = await app.db.sent_posts(2)
+        assert p.buttons == [[{"t": "Join", "u": "https://t.me/animech"}, {"t": "Site", "u": "https://example.com/x"}], [{"t": "More", "u": "https://t.me/moviech"}]]
+        assert p.entities == [{"k": "bold", "o": 0, "l": 5}]
+        await app.db.close()
+
+    run(go())
+
+
+def test_a_photo_with_a_caption(tmp_path):
+    async def go():
+        app = await make_app(tmp_path)
+        msg = SimpleNamespace(raw_text="New poster", entities=[], media=photo_media(5), grouped_id=None, forward=None)
+        await broadcast(app, "/broadcast 30m", "New poster", message=msg)
+        files = [e for e in posted(app) if e[0] == "file"]
+        assert len(files) == 2 and files[0][3]["caption"] == "New poster"
+        got = app.tg.registry[2].msgs[1]
+        assert got.message == "New poster" and isinstance(got.media, types.MessageMediaPhoto)
+        (p,) = await app.db.sent_posts(2)
+        assert p.media_kind == "photo" and p.media_file_id.startswith("tl:") and p.text == "New poster"
+        await app.db.close()
+
+    run(go())
+
+
+def test_a_channel_that_refuses_does_not_stop_the_others(tmp_path):
+    async def go():
+        app = await make_app(tmp_path)
+        real = app.tg.send_message
+
+        async def picky(peer, message="", **kw):
+            if getattr(peer, "channel_id", None) == 2:
+                raise errors.ChatWriteForbiddenError(request=None)
+            return await real(peer, message, **kw)
+
+        app.tg.send_message = picky
+        await broadcast(app, "/broadcast 1h")
+        report = norm(app.out.last_text)
+        assert "sent to 1 of 2" in report and "✅ Anime Channel" in report
+        assert "❌ Movie Channel" in report and "ChatWriteForbiddenError" in report
+        assert [r.channel_id for r in await app.db.pending_deletes()] == [1]  # only what was sent expires
+        assert await app.db.sent_posts(2) == []
+        await app.db.close()
+
+    run(go())
+
+
+def test_only_the_owner_and_only_good_input(tmp_path):
+    async def go():
+        app = await make_app(tmp_path, admins=frozenset({7}))
+        app.uid = 7
+        await app.text("/broadcast 1h")
+        assert "private" in app.out.last_text and 7 not in app.ctx.state
+        app.uid = 1
+        await app.text("/broadcast soon")
+        assert "can't read" in norm(app.out.last_text) and 1 not in app.ctx.state
+        await app.text("/broadcast 20s")
+        assert "don't know the unit" in norm(app.out.last_text)
+        await app.text("/broadcast 400d")
+        assert "1 year" in app.out.last_text
+        # an admin cannot press the owner's buttons either
+        await app.text("/broadcast 1h")
+        await app.text("Hi")
+        await app.press(app.out.callback_data("No buttons"))
+        confirm = app.out.callback_data("Send to all channels")
+        app.uid = 7
+        await app.press(confirm)
+        assert "Only the bot owner" in app.out.last_text and posted(app) == []
+        await app.db.close()
+
+    run(go())
+
+
+def test_nothing_is_sent_without_a_yes_and_a_button_works_once(tmp_path):
+    async def go():
+        app = await make_app(tmp_path)
+        await app.text("/broadcast")
+        await app.text("Hi")
+        await app.press(app.out.callback_data("No buttons"))
+        await app.press(app.out.callback_data("Cancel"))
+        assert "Cancelled" in app.out.last_text and posted(app) == []
+
+        await app.text("/broadcast")
+        await app.text("Hi")
+        await app.press(app.out.callback_data("No buttons"))
+        send = app.out.callback_data("Send to all channels")
+        await app.press(send)
+        await app.press(send)
+        assert "expired" in app.out.last_text
+        assert len(posted(app)) == 2
+        await app.db.close()
+
+    run(go())
+
+
+def test_wrong_buttons_and_a_busy_bot_are_explained(tmp_path):
+    async def go():
+        app = await make_app(tmp_path)
+        await app.text("/broadcast 1h")
+        await app.text("Hi")
+        await app.text("no link here")
+        assert "Try again" in norm(app.out.last_text)
+        await app.text("Go - https://t.me/x")
+        send = app.out.callback_data("Send to all channels")
+        await app.ctx.lock.acquire()
+        await app.press(send)
+        assert "still running" in app.out.last_text and posted(app) == []
+        app.ctx.lock.release()
+        await app.press(send)
+        assert len(posted(app)) == 2
+        await app.db.close()
+
+    run(go())
+
+
+def test_no_channels_no_broadcast(tmp_path):
+    async def go():
+        app = App(tmp_path, edit_delay=0)
+        await app.start()
+        await app.text("/broadcast 1h")
+        assert "No channels yet" in app.out.last_text
+        await app.db.close()
+
+    run(go())
+
+
+# ------------------------------------------------------------------------------------ deleting
+async def timed_broadcast(app, command="/broadcast 1h"):
+    await broadcast(app, command)
+    return utcnow()
+
+
+def test_when_the_time_is_up_the_message_goes_from_the_channels_and_my_posts(tmp_path):
+    async def go():
+        app = await make_app(tmp_path)
+        now = await timed_broadcast(app)
+        ex = app.ctx.expirer
+
+        early = await ex.sweep(now=now + timedelta(minutes=59))
+        assert early["deleted"] == 0 and 1 in app.tg.registry[1].msgs
+        assert len(await app.db.sent_posts(1)) == 1
+
+        done = await ex.sweep(now=now + timedelta(hours=1, minutes=1))
+        assert done["deleted"] == 2 and done["retry"] == 0 and done["failed"] == 0
+        assert app.tg.registry[1].msgs == {} and app.tg.registry[2].msgs == {}
+        assert await app.db.sent_posts(1) == [] and await app.db.sent_posts(2) == []
+        assert await app.db.pending_deletes() == []
+        assert ex._next is None  # nothing left: the loop sleeps until something new is scheduled
+        await app.db.close()
+
+    run(go())
+
+
+def test_other_posts_are_not_touched_and_a_message_deleted_by_hand_is_fine(tmp_path):
+    async def go():
+        app = await make_app(tmp_path)
+        keep = await app.db.create_post(channel_id=1, message_id=50, status="sent", source="bot", text="keep me", entities=[], buttons=[])
+        now = await timed_broadcast(app)
+        app.tg.registry[2].msgs.pop(1)  # an admin deleted the broadcast in channel 2 already
+        done = await app.ctx.expirer.sweep(now=now + timedelta(hours=2))
+        assert done["deleted"] == 2
+        assert (await app.db.get_post(keep.id)) is not None
+        assert [p.message_id for p in await app.db.sent_posts(1)] == [50]
+        assert await app.db.sent_posts(2) == []
+        await app.db.close()
+
+    run(go())
+
+
+def test_a_restart_loses_nothing(tmp_path):
+    async def go():
+        app = await make_app(tmp_path)
+        now = await timed_broadcast(app, "/broadcast 5d")
+        due = now + timedelta(days=5)
+        fresh = Expirer(app.ctx)  # a new process: it knows nothing but the database
+        assert fresh._next is None
+        await fresh.load()
+        assert abs((fresh._next - due).total_seconds()) < 30
+        # it was off for a week: everything that came due is deleted when it is back
+        await fresh.sweep(now=now + timedelta(days=7))
+        assert app.tg.registry[1].msgs == {} and app.tg.registry[2].msgs == {}
+        assert await app.db.sent_posts(1) == []
+        await app.db.close()
+
+    run(go())
+
+
+def test_the_background_loop_wakes_when_something_is_scheduled_and_when_it_is_due(tmp_path):
+    async def go():
+        app = await make_app(tmp_path)
+        ex = app.ctx.expirer
+        ex.start()
+        await asyncio.sleep(0.05)  # idle: nothing to do, nothing polled
+        msg = app.tg._new("soon gone", None, None)
+        when = utcnow() + timedelta(seconds=0.4)
+        await app.db.schedule_deletes([dict(channel_id=1, message_id=msg.id, delete_at=when, group_id="g", created_by=1)])
+        ex.notify(when)
+        for _ in range(60):
+            if msg.id not in app.tg.msgs:
+                break
+            await asyncio.sleep(0.05)
+        assert msg.id not in app.tg.msgs
+        await ex.stop()
+        await app.db.close()
+
+    run(go())
+
+
+def test_something_already_overdue_is_deleted_at_start(tmp_path):
+    async def go():
+        app = await make_app(tmp_path)
+        msg = app.tg._new("overdue", None, None)
+        await app.db.schedule_deletes([dict(channel_id=1, message_id=msg.id, delete_at=utcnow() - timedelta(hours=3), group_id="g", created_by=1)])
+        ex = Expirer(app.ctx)
+        ex.start()
+        for _ in range(60):
+            if msg.id not in app.tg.msgs:
+                break
+            await asyncio.sleep(0.05)
+        assert msg.id not in app.tg.msgs
+        await ex.stop()
+        await app.db.close()
+
+    run(go())
+
+
+def test_a_running_job_makes_it_wait(tmp_path):
+    async def go():
+        app = await make_app(tmp_path)
+        now = await timed_broadcast(app)
+        ex = app.ctx.expirer
+        await app.ctx.lock.acquire()
+        later = now + timedelta(hours=2)
+        out = await ex.sweep(now=later)
+        assert out["waiting"] and out["deleted"] == 0 and 1 in app.tg.registry[1].msgs
+        assert 50 <= (ex._next - later).total_seconds() <= 70
+        app.ctx.lock.release()
+        out = await ex.sweep(now=later)
+        assert out["deleted"] == 2
+        await app.db.close()
+
+    run(go())
+
+
+def test_what_the_bot_cannot_delete_is_retried_then_reported(tmp_path):
+    async def go():
+        app = await make_app(tmp_path, channels=1)
+        now = await timed_broadcast(app)
+        app.tg.undeletable = {1}  # Telegram refuses (an old post)
+        ex = app.ctx.expirer
+        t = now + timedelta(hours=2)
+        for attempt in range(1, MAX_ATTEMPTS):
+            out = await ex.sweep(now=t)
+            assert out["retry"] == 1 and out["failed"] == 0, attempt
+            (row,) = await app.db.pending_deletes()
+            assert row.attempts == attempt and as_utc(row.delete_at) > t
+            t = as_utc(row.delete_at) + timedelta(seconds=1)
+        dms_before = len([e for e in app.tg.sent if not isinstance(e[1], types.InputPeerChannel)])
+        out = await ex.sweep(now=t)
+        assert out["failed"] == 1
+        assert await app.db.pending_deletes() == []
+        (row,) = await app.db.pending_deletes(status="failed")
+        assert row.attempts == MAX_ATTEMPTS and row.last_error
+        # the person who sent it is told; the post stays in My posts because it is still in the channel
+        told = [e for e in app.tg.sent if not isinstance(e[1], types.InputPeerChannel)][dms_before:]
+        assert told and "could not delete the expired broadcast" in told[-1][2]
+        assert len(await app.db.sent_posts(1)) == 1
+        assert ex._next is None
+        await app.db.close()
+
+    run(go())
+
+
+def test_the_userbot_deletes_what_the_bot_may_not(tmp_path):
+    async def go():
+        app = await make_app(tmp_path, channels=1)
+        now = await timed_broadcast(app)
+        app.tg.undeletable = {1}
+        calls, closed = [], []
+
+        class StubUserbot:
+            enabled = True
+
+            async def fallback_for(self, ch):
+                async def fn(ids):
+                    calls.append(list(ids))
+                    for i in ids:
+                        app.tg.msgs.pop(i, None)
+
+                async def close():
+                    closed.append(1)
+
+                return fn, SimpleNamespace(close=close)
+
+        app.ctx.userbot = StubUserbot()
+        out = await app.ctx.expirer.sweep(now=now + timedelta(hours=2))
+        assert out["deleted"] == 1 and calls == [[1]] and closed == [1]
+        assert app.tg.msgs == {} and await app.db.sent_posts(1) == []
+        await app.db.close()
+
+    run(go())
+
+
+def test_a_channel_taken_out_of_the_bot_is_reported_not_retried_forever(tmp_path):
+    async def go():
+        app = await make_app(tmp_path, channels=1)
+        now = await timed_broadcast(app)
+        await app.db.forget_posts(1, [p.message_id for p in await app.db.sent_posts(1)])  # a real database refuses to drop a channel its posts still point at
+        async with app.db.Session() as s:
+            await s.delete(await s.get(Channel, 1))
+            await s.commit()
+        out = await app.ctx.expirer.sweep(now=now + timedelta(hours=2))
+        assert out["failed"] == 1 and out["deleted"] == 0
+        assert await app.db.pending_deletes() == []
+        await app.db.close()
+
+    run(go())

tests/test_config.py

@@ -21,7 +21,7 @@ REQUIRED = {"API_ID": "12345", "API_HASH": "abcdef0123456789", "BOT_TOKEN": "123
 def env_names() -> list:
     """Every environment variable app/config.py reads."""
     src = (ROOT / "app" / "config.py").read_text()
-    names = set(re.findall(r'(?:getenv|_ints|_bool|_chat_id|_float)\(\s*"([A-Z][A-Z0-9_]+)"', src))
+    names = set(re.findall(r'(?:getenv|_ints|_bool|_chat_id|_float|_whole)\(\s*"([A-Z][A-Z0-9_]+)"', src))
     names |= set(re.findall(r'environ\[\s*"([A-Z][A-Z0-9_]+)"', src))
     return sorted(names)
 
@@ -113,6 +113,26 @@ def test_api_id_must_be_a_number(env):
     assert "API_ID must be a number" in str(e.value)
 
 
+def test_service_message_and_sync_settings(env):
+    cfg = load_config()
+    assert cfg.delete_service_messages is True and cfg.sync_interval_minutes == 60  # both are on unless somebody says otherwise
+    env.setenv("DELETE_SERVICE_MESSAGES", "false")
+    env.setenv("SYNC_INTERVAL_MINUTES", "15")
+    cfg = load_config()
+    assert cfg.delete_service_messages is False and cfg.sync_interval_minutes == 15
+    env.setenv("DELETE_SERVICE_MESSAGES", "yes")
+    assert load_config().delete_service_messages is True
+
+
+@pytest.mark.parametrize(
+    "raw,expected",
+    [("0", 0), ("-5", 0), ("30", 30), ("2.9", 2), ("  45 ", 45), ("", 60), ("abc", 60), ("inf", 60), ("-inf", 60), ("nan", 60)],
+)
+def test_sync_interval_minutes_never_stops_the_bot_from_starting(env, raw, expected):
+    env.setenv("SYNC_INTERVAL_MINUTES", raw)
+    assert load_config().sync_interval_minutes == expected  # 0 = no automatic checks; nonsense = the default
+
+
 def test_edit_delay_has_a_floor(env):
     env.setenv("EDIT_DELAY", "0.1")
     assert load_config().edit_delay == 0.3
@@ -202,10 +222,12 @@ def test_helper_scripts_compile_and_the_userbot_one_stands_alone():
 def test_the_testing_commands_are_gone_everywhere():
     names = [c for c, _ in COMMANDS]
     assert "testedit" not in names and "selftest" not in names
-    assert {"new", "posts", "repost", "replace", "undo", "userbot", "testerror", "help"} <= set(names)
+    assert {"new", "posts", "repost", "replace", "undo", "userbot", "help"} <= set(names)
+    assert {"autopost", "broadcast", "handleswap", "fixlinks", "shift", "sync"} <= set(names)
+    assert "testerror" not in names
     for path in list((ROOT / "app").rglob("*.py")) + [ROOT / "bot.py"]:
         text = path.read_text().lower()
-        assert "testedit" not in text and "selftest" not in text, path
+        assert "testedit" not in text and "selftest" not in text and "testerror" not in text, path
 
 
 def test_every_menu_command_has_a_handler(tmp_path):
@@ -223,9 +245,12 @@ def test_every_menu_command_has_a_handler(tmp_path):
 def test_the_help_text_lists_the_commands_that_exist():
     from app.handlers.basic import HELP
 
-    for name in ("new", "posts", "repost", "replace", "undo", "userbot", "testerror", "export", "addchannel", "channels"):
+    for name in (
+        "new", "posts", "repost", "replace", "undo", "userbot", "export", "addchannel", "channels",
+        "autopost", "broadcast", "handleswap", "fixlinks", "shift", "sync",
+    ):
         assert f"/{name}" in HELP, name
-    assert "--channel" not in HELP
+    assert "--channel" not in HELP and "/testerror" not in HELP
 
 
 # ------------------------------------------------------------------------------------------ health port

tests/test_errorlog.py

@@ -292,7 +292,7 @@ def test_the_logging_handler_forwards_errors_with_their_traceback():
 
 
 def test_a_failing_handler_in_the_bot_reaches_the_log_chat_end_to_end(tmp_path):
-    """/testerror raises on purpose inside a real handler; guard() logs it; the log handler sends it."""
+    """A real handler fails; guard() logs it; the log handler sends it as a code block with the whole chain."""
 
     async def go():
         client = FakeSender()
@@ -303,17 +303,21 @@ def test_a_failing_handler_in_the_bot_reaches_the_log_chat_end_to_end(tmp_path):
         app = App(tmp_path)
         await app.start()
         app.ctx.reporter = rep
+
+        async def broken():
+            raise boom()
+
+        app.db.list_channels = broken  # /channels now fails the way a database outage would
         try:
-            await app.text("/testerror")
+            await app.text("/channels")
             await asyncio.sleep(0.05)
             await drain(rep)
         finally:
             logging.getLogger().removeHandler(handler)
-        assert "Raising a test error now" in norm(app.out.log[0][1])
-        assert "Something went wrong: RuntimeError" in norm(app.out.last_text)  # the owner is told too
+        assert "Something went wrong: RuntimeError" in norm(app.out.last_text)  # the user is told too
         text = client.texts[0]
-        assert text.startswith("Unhandled exception in handler h_testerror (user 1):\n\nTraceback")
-        assert "KeyError: 'test'" in text and "RuntimeError: Test error from /testerror" in text
+        assert text.startswith("Unhandled exception in handler h_channels (user 1):\n\nTraceback")
+        assert "KeyError: 'inner'" in text and "RuntimeError: outer" in text
         assert "The above exception was the direct cause of the following exception:" in text
         await app.db.close()
         await rep.stop()
@@ -321,46 +325,28 @@ def test_a_failing_handler_in_the_bot_reaches_the_log_chat_end_to_end(tmp_path):
     asyncio.run(go())
 
 
-def test_testerror_task_makes_a_background_task_fail(tmp_path):
+def test_a_forgotten_background_task_that_fails_reaches_the_log_chat(tmp_path):
     async def go():
         client = FakeSender()
         rep = reporter(client)
         rep.start()
         asyncio.get_running_loop().set_exception_handler(rep.loop_exception_handler)
-        app = App(tmp_path)
-        await app.start()
-        app.ctx.reporter = rep
-        await app.text("/testerror task")
-        assert "background task will fail" in norm(app.out.last_text)
+
+        async def fails():
+            raise RuntimeError("background failure")
+
+        asyncio.get_running_loop().create_task(fails())  # nobody awaits it
         await asyncio.sleep(0.1)
         gc.collect()
         await asyncio.sleep(0.05)
         await drain(rep)
         assert client.texts[0].startswith("Unhandled exception in background task <unnamed>:")
-        assert "RuntimeError: Test error from /testerror task" in client.texts[0]
-        await app.db.close()
+        assert "RuntimeError: background failure" in client.texts[0]
         await rep.stop()
 
     asyncio.run(go())
 
 
-def test_testerror_is_for_the_owner_and_says_when_the_log_is_off(tmp_path):
-    async def go():
-        app = App(tmp_path, admins=frozenset({7}))
-        await app.start()
-        await app.text("/testerror")  # no reporter at all
-        assert "switched off" in norm(app.out.last_text)
-        app.ctx.reporter = reporter(FakeSender(), chat_id=None)
-        await app.text("/testerror")
-        assert "switched off" in norm(app.out.last_text)
-        app.uid = 7
-        await app.text("/testerror")
-        assert "private" in norm(app.out.last_text)
-        await app.db.close()
-
-    asyncio.run(go())
-
-
 def test_a_crash_of_the_whole_bot_is_sent_directly_even_when_disconnected():
     async def go():
         client = FakeSender(connected=False, authorized=False)

tests/test_handleswap.py

@@ -0,0 +1,260 @@
+"""/handleswap @bro @sis: the username written in posts changes - links, hyperlinks and button links do not."""
+import asyncio
+
+import pytest
+from telethon import types
+
+from app.handleswap import compute_handle_changes, find_mention_spans, mention_regex
+from app.handlers.replace import parse_replace_args
+from app.replace_engine import ScanOptions, apply_channel, scan_channel, undo_batch
+from app.tgutil import button_url, is_callback_button
+from telethon.helpers import add_surrogate
+
+from .dbutil import db_url_for, fresh_db
+from .fakes import FakeClient, cb_btn, make_msg, markup, url_btn
+from .harness import App, norm
+
+
+def run(coro):
+    return asyncio.run(coro)
+
+
+def change(text, entities=None, mk=None):
+    return compute_handle_changes(make_msg(1, text, entities, mk), "bro", "sis")
+
+
+def utf16(s, start, length):
+    return s.encode("utf-16-le")[start * 2 : (start + length) * 2].decode("utf-16-le")
+
+
+# ------------------------------------------------------------------------------- what counts as a username
+def test_only_whole_usernames_are_found():
+    rx = mention_regex("bro")
+    found = lambda s: [m.group(0) for m in rx.finditer(s)]  # noqa: E731
+    assert found("Join @bro now") == ["@bro"]
+    assert found("@bro") == ["@bro"] and found("(@BRO)!") == ["@BRO"] and found("a,@bro.") == ["@bro"]
+    assert found("two: @bro @Bro") == ["@bro", "@Bro"]
+    assert found("Contact:@bro") == ["@bro"]
+    # parts of other things are not usernames
+    assert found("@bro_bot @brother @bro2") == []
+    assert found("mail me: a@bro.com") == []
+    assert found("https://example.com/x?u=@bro") == []
+    assert found("t.me/bro and https://t.me/bro?start=1") == []
+
+
+def test_a_username_in_text_is_swapped_and_the_rest_stays():
+    text = "Join @bro and @Bro_bot, mail a@bro.com, https://t.me/bro?x=@bro, see t.me/bro, (@BRO)!"
+    ch, skipped = change(text)
+    assert ch.n_typed == 2 and ch.n_links == 0 and ch.n_buttons == 0 and skipped == 0
+    assert ch.new_text == "Join @sis and @Bro_bot, mail a@bro.com, https://t.me/bro?x=@bro, see t.me/bro, (@sis)!"
+    assert change("nothing to see here")[0] is None
+    assert change("only links: t.me/bro https://t.me/bro?start=a")[0] is None
+
+
+def test_formatting_stays_in_place_when_the_name_gets_longer_or_shorter():
+    text = "Hi @bro, welcome to @bro's place 😀 and @bro"
+    ents = [
+        types.MessageEntityBold(3, 4),  # exactly the first @bro
+        types.MessageEntityMention(3, 4),
+        types.MessageEntityItalic(9, 7),  # ", welcome"[..] runs across text after it
+        types.MessageEntityCode(len(text.encode("utf-16-le")) // 2 - 4, 4),  # the last @bro
+    ]
+    ch, _ = compute_handle_changes(make_msg(1, text, ents), "bro", "sister")
+    assert ch.new_text == "Hi @sister, welcome to @sister's place 😀 and @sister"
+    bold, mention, italic, code = ch.new_entities
+    assert utf16(ch.new_text, bold.offset, bold.length) == "@sister" == utf16(ch.new_text, mention.offset, mention.length)
+    assert utf16(ch.new_text, code.offset, code.length) == "@sister"
+    assert italic.offset == 3 + 7 + 2 and italic.length == 7  # starts after the longer name: ", welcome" begins 3+7
+    # shorter name
+    ch, _ = compute_handle_changes(make_msg(1, "x @brother y", [types.MessageEntityBold(0, 1)]), "brother", "bo")
+    assert ch.new_text == "x @bo y" and ch.new_entities[0].offset == 0
+
+
+def test_links_hyperlinks_and_emails_are_left_alone():
+    text = "@bro writes: https://example.com/p?u=@bro, mail x@bro.com or see @bro"
+    url_start = len("@bro writes: ")
+    url = "https://example.com/p?u=@bro"
+    ents = [
+        types.MessageEntityUrl(url_start, len(url)),
+        types.MessageEntityEmail(text.index("x@bro.com"), len("x@bro.com")),
+        types.MessageEntityTextUrl(len(text) - 4, 4, "https://t.me/bro"),  # the last "@bro" is a hyperlink
+    ]
+    ch, skipped = compute_handle_changes(make_msg(1, text, ents), "bro", "sis")
+    assert ch.n_typed == 1 and ch.new_text.startswith("@sis writes: https://example.com/p?u=@bro, mail x@bro.com")
+    assert ch.new_text.endswith("or see @bro")  # the hyperlinked @bro is a hyperlink: not touched
+    assert skipped == 1
+    link = [e for e in ch.new_entities if isinstance(e, types.MessageEntityTextUrl)][0]
+    assert link.url == "https://t.me/bro"  # and its link is exactly as it was
+
+
+def test_button_names_change_but_their_links_do_not():
+    mk = markup(
+        [url_btn("Join @bro", "https://t.me/bro?start=1"), cb_btn("👍 @bro", b"r")],
+        [url_btn("Open https://t.me/bro", "https://t.me/bro")],
+    )
+    ch, _ = change("hello", mk=mk)
+    assert (ch.n_typed, ch.n_buttons, ch.text_changed, ch.callbacks) == (0, 1, False, True)
+    rows = ch.new_markup.rows
+    assert rows[0].buttons[0].text == "Join @sis" and button_url(rows[0].buttons[0]) == "https://t.me/bro?start=1"
+    assert is_callback_button(rows[0].buttons[1]) and rows[0].buttons[1].text == "👍 @bro"  # another bot's button
+    assert rows[1].buttons[0].text == "Open https://t.me/bro"
+    # no username anywhere: nothing to do and the keyboard object is not rebuilt
+    assert change("hello", mk=markup([url_btn("Go", "https://t.me/bro")]))[0] is None
+
+
+def test_find_mention_spans_counts_what_it_skips():
+    sur = add_surrogate("@bro and @bro")
+    spans, skipped = find_mention_spans(sur, [types.MessageEntityTextUrl(9, 4, "https://x.y")], "bro")
+    assert spans == [(0, 4)] and skipped == 1
+
+
+def test_the_arguments():
+    assert parse_replace_args("@bro @sis", handles=True)[:2] == ("bro", "sis")
+    assert parse_replace_args("bro sis --channel @a --last 50", handles=True)[2] == {"channel": "@a", "last": 50, "posts": False, "typed": None}
+    for bad in ("@bro", "@bro @bro", "@bro @sis --posts", "@bro @sis --no-typed", "@b!ro @sis"):
+        with pytest.raises(ValueError):
+            parse_replace_args(bad, handles=True)
+
+
+# ------------------------------------------------------------------------------- the engine
+def build():
+    return {
+        1: make_msg(1, "Join @bro today", [types.MessageEntityBold(5, 4)], out=True),
+        2: make_msg(2, "Episodes", markup=markup([url_btn("Download @bro", "https://t.me/bro?start=A"), cb_btn("👍 3", b"r")]), out=True),
+        3: make_msg(3, "link only https://t.me/bro?start=B", [types.MessageEntityUrl(10, 24)]),
+        4: make_msg(4, "by @bro_bot"),
+        5: make_msg(5, "@BRO wrote this", media=None),  # someone else's post
+        7: types.MessageEmpty(id=7, peer_id=types.PeerChannel(1)),
+    }
+
+
+async def engine_setup(tmp_path, msgs):
+    db = await fresh_db(db_url_for(tmp_path))
+    await db.save_channel(1, 5, "Test", None, 99)
+    return db, await db.get_channel(1), FakeClient(msgs, pts=8)
+
+
+def test_scan_apply_and_undo(tmp_path):
+    async def go():
+        msgs = build()
+        db, ch, client = await engine_setup(tmp_path, msgs)
+        opts = ScanOptions("bro", "sis", mode="handles")
+        scan = await scan_channel(client, ch, opts, can_edit_others=True)
+        assert set(scan.infos) == {1, 2, 5}
+        assert (scan.infos[1].typed, scan.infos[2].buttons, scan.infos[2].callbacks) == (1, 1, True)
+        assert scan.foreign == 1
+
+        await db.create_batch("hs1", "bro", "sis", 99)
+        res = await apply_channel(client, db, scan, opts, "hs1", 99, edit_delay=0)
+        assert (res.edited, res.failed) == (3, [])
+        assert msgs[1].message == "Join @sis today" and (msgs[1].entities[0].offset, msgs[1].entities[0].length) == (5, 4)
+        assert msgs[5].message == "@sis wrote this"
+        rows = msgs[2].reply_markup.rows
+        assert rows[0].buttons[0].text == "Download @sis" and button_url(rows[0].buttons[0]) == "https://t.me/bro?start=A"
+        assert is_callback_button(rows[0].buttons[1])  # the other bot's button survives the text edit
+        assert msgs[2].message == "Episodes"  # only the button name changed
+        assert msgs[3].message == "link only https://t.me/bro?start=B" and msgs[4].message == "by @bro_bot"
+
+        # My posts follows
+        p1 = await db.find_by_message(1, 1)
+        assert p1.text == "Join @sis today"
+        p2 = await db.find_by_message(1, 2)
+        assert p2.buttons[0][0] == {"t": "Download @sis", "u": "https://t.me/bro?start=A"}
+
+        back = await undo_batch(client, db, "hs1", edit_delay=0)
+        assert (back.restored, back.failed) == (3, [])
+        assert msgs[1].message == "Join @bro today" and msgs[5].message == "@BRO wrote this"
+        assert msgs[2].reply_markup.rows[0].buttons[0].text == "Download @bro"
+        await db.close()
+
+    run(go())
+
+
+# --------------------------------------------------------------------------------- the command
+async def flow_app(tmp_path, **cfg):
+    app = App(tmp_path, **cfg)
+    await app.start()
+    await app.db.save_channel(1, 5, "Anime Channel", "animech", 1)
+    app.tg.msgs = build()
+    app.tg._top = 8
+    return app
+
+
+def test_the_whole_command_with_undo(tmp_path):
+    async def go():
+        app = await flow_app(tmp_path)
+        await app.text("/handleswap")
+        assert "Usage: /handleswap @old @new" in norm(app.out.last_text)
+        await app.text("/handleswap @bro @sis --posts")
+        assert "Unknown option --posts" in app.out.last_text
+        await app.text("/handleswap @bro")
+        assert "old and the new username" in app.out.last_text
+
+        await app.text("/handleswap @bro @sis")
+        shown = norm(app.out.last_text)
+        assert "swap the username @bro → @sis" in shown and "3 post(s) to change" in shown
+        assert "3 mention(s) in the text" not in shown and "2 mention(s) in the text, 1 in button names" in shown
+        assert "Nothing has been changed yet" in shown and "Apply to 3 post(s)" in str(app.out.last_buttons())
+        assert app.tg.msgs[1].message == "Join @bro today"  # still untouched
+
+        await app.press(app.out.callback_data("Apply to 3"))
+        done = norm(app.out.last_text)
+        assert "Done - username swap @bro → @sis" in done and "edited: 3" in done
+        assert app.tg.msgs[1].message == "Join @sis today" and app.tg.msgs[5].message == "@sis wrote this"
+        assert app.tg.msgs[2].reply_markup.rows[0].buttons[0].text == "Download @sis"
+        assert "Undo this swap" in str(app.out.last_buttons())
+
+        await app.text("/undo")
+        assert "Undo the username swap @bro → @sis" in norm(app.out.last_text) and "3 post(s)" in norm(app.out.last_text)
+        await app.press(app.out.callback_data("Yes, undo"))
+        assert "Restored 3 post(s)" in norm(app.out.last_text)
+        assert app.tg.msgs[1].message == "Join @bro today" and app.tg.msgs[2].reply_markup.rows[0].buttons[0].text == "Download @bro"
+        await app.db.close()
+
+    run(go())
+
+
+def test_nothing_to_swap_and_only_the_owner(tmp_path):
+    async def go():
+        app = await flow_app(tmp_path)
+        await app.text("/handleswap @nobody @sis")
+        assert "Nothing to change" in app.out.last_text and "Apply" not in str(app.out.last_buttons())
+        app.uid = 99
+        await app.text("/handleswap @bro @sis")
+        assert "private" in app.out.last_text
+        app.uid = 1
+        await app.text("/undo")
+        assert "nothing to undo" in app.out.last_text
+        await app.db.close()
+
+    run(go())
+
+
+def test_it_can_be_limited_to_one_channel(tmp_path):
+    async def go():
+        app = await flow_app(tmp_path)
+        await app.text("/handleswap @bro @sis --channel @nosuchchannel")
+        assert "isn't registered" in norm(app.out.last_text)
+        await app.text("/handleswap @bro @sis --channel @animech")
+        assert "Apply to 3 post(s)" in str(app.out.last_buttons())
+        await app.db.close()
+
+    run(go())
+
+
+def test_replace_still_works_after_the_refactor(tmp_path):
+    async def go():
+        app = await flow_app(tmp_path)
+        app.tg.msgs[3].reply_markup = markup([url_btn("Get", "https://t.me/bro?start=1")])
+        await app.text("/replace @bro @sis")
+        shown = norm(app.out.last_text)
+        assert "replace @bro → @sis" in shown and "Apply to" in str(app.out.last_buttons())
+        await app.press(app.out.callback_data("Apply to"))
+        assert "Done - @bro → @sis" in norm(app.out.last_text)
+        assert button_url(app.tg.msgs[3].reply_markup.rows[0].buttons[0]) == "https://t.me/sis?start=1"
+        assert app.tg.msgs[1].message == "Join @bro today"  # /replace never touches a plain username
+        await app.text("/undo")
+        assert "Undo the replace @bro → @sis" in norm(app.out.last_text)
+        await app.db.close()
+
+    run(go())

tests/test_postlinks.py

@@ -0,0 +1,155 @@
+"""Links to other posts of the same channel: found in all three places and pointed at the new message ids."""
+from telethon import types
+from telethon.helpers import add_surrogate
+
+from app.postlinks import PostLinks, parse_ref, relink_message, resolve_chain
+from app.tgutil import button_url
+
+from .fakes import make_msg, markup, url_btn
+
+
+def links(ids=None, username="One_Piece_inEnglish", cid=1234567):
+    return PostLinks(username=username, channel_id=cid, ids=dict(ids or {}))
+
+
+def utf16(text, ent):
+    raw = text.encode("utf-16-le")
+    return raw[ent.offset * 2 : (ent.offset + ent.length) * 2].decode("utf-16-le")
+
+
+# ------------------------------------------------------------------------------------------- parsing
+def test_the_forms_of_a_post_link_are_recognised():
+    r = parse_ref("https://t.me/One_Piece_inEnglish/115")
+    assert (r.name, r.cid, r.msg_id) == ("one_piece_inenglish", None, 115)
+    assert "https://t.me/One_Piece_inEnglish/115"[r.start : r.end] == "115"
+    r = parse_ref("t.me/c/1234567/56?single")
+    assert (r.name, r.cid, r.msg_id) == (None, 1234567, 56)
+    assert parse_ref("http://telegram.me/somechan/7#x").msg_id == 7
+    assert parse_ref("https://t.me/s/somechan/7").msg_id == 7
+    assert parse_ref("https://telegram.dog/somechan/7/2").msg_id == 7  # only the first number is the post
+    r = parse_ref("tg://resolve?domain=somechan&post=9")
+    assert (r.name, r.msg_id) == ("somechan", 9)
+    r = parse_ref("tg://resolve?post=9&domain=somechan")
+    assert (r.name, r.msg_id) == ("somechan", 9)
+    r = parse_ref("tg://privatepost?channel=1234567&post=9")
+    assert (r.cid, r.msg_id) == (1234567, 9)
+
+
+def test_links_that_are_not_post_links_are_not_touched():
+    for url in (
+        "https://t.me/somechan",
+        "https://t.me/somechan?start=ABC123",
+        "https://t.me/+AbCdEfGhIj",
+        "https://t.me/addstickers/pack",
+        "https://example.com/somechan/12",
+        "https://t.me/c/1234567",
+        "tg://resolve?domain=somechan",
+        "",
+        None,
+    ):
+        assert parse_ref(url) is None, url
+
+
+def test_only_links_into_this_channel_count():
+    pl = links({115: 400})
+    assert pl.ref_of("https://t.me/one_piece_inenglish/115")  # a username is not case sensitive
+    assert pl.ref_of("https://t.me/c/1234567/115")
+    assert pl.ref_of("https://t.me/c/999/115") is None
+    assert pl.ref_of("https://t.me/another_channel/115") is None
+    private = PostLinks(username=None, channel_id=1234567, ids={115: 400})
+    assert private.ref_of("https://t.me/One_Piece_inEnglish/115") is None
+    assert private.remap_url("https://t.me/c/1234567/115") == "https://t.me/c/1234567/400"
+
+
+def test_only_the_digits_of_the_post_id_change():
+    pl = links({115: 1042, 7: 8})
+    assert pl.remap_url("https://t.me/One_Piece_inEnglish/115") == "https://t.me/One_Piece_inEnglish/1042"
+    assert pl.remap_url("https://t.me/One_Piece_inEnglish/115?single") == "https://t.me/One_Piece_inEnglish/1042?single"
+    assert pl.remap_url("t.me/c/1234567/7?comment=7") == "t.me/c/1234567/8?comment=7"
+    assert pl.remap_url("tg://resolve?domain=One_Piece_inEnglish&post=115") == "tg://resolve?domain=One_Piece_inEnglish&post=1042"
+    assert pl.remap_url("tg://resolve?post=115&domain=One_Piece_inEnglish&x=1") == "tg://resolve?post=1042&domain=One_Piece_inEnglish&x=1"
+    assert pl.remap_url("tg://privatepost?channel=1234567&post=7") == "tg://privatepost?channel=1234567&post=8"
+    # a post nobody copied, another channel, a bot link: unchanged
+    assert pl.remap_url("https://t.me/One_Piece_inEnglish/116") == "https://t.me/One_Piece_inEnglish/116"
+    assert pl.remap_url("https://t.me/other/115") == "https://t.me/other/115"
+    assert pl.remap_url("https://t.me/goku?start=115") == "https://t.me/goku?start=115"
+
+
+# --------------------------------------------------------------------------------------- the text
+def test_typed_links_are_rewritten_and_the_formatting_after_them_moves_along():
+    text = "😀 see https://t.me/One_Piece_inEnglish/9 and bold t.me/c/1234567/10?single end"
+    bold = types.MessageEntityBold(text.index("bold") + 0, 4)
+    bold.offset = len(text[: text.index("bold")].encode("utf-16-le")) // 2  # offsets count UTF-16 units, the emoji is 2
+    url_ent = types.MessageEntityUrl(len("😀 see ".encode("utf-16-le")) // 2, len("https://t.me/One_Piece_inEnglish/9"))
+    pl = links({9: 1234, 10: 12})
+    new_text, ents, n_typed, n_hyper = pl.remap_text(text, [bold, url_ent])
+    assert new_text == "😀 see https://t.me/One_Piece_inEnglish/1234 and bold t.me/c/1234567/12?single end"
+    assert (n_typed, n_hyper) == (2, 0)
+    b, u = ents
+    assert utf16(new_text, b) == "bold"  # the bold span moved with its text
+    assert utf16(new_text, u) == "https://t.me/One_Piece_inEnglish/1234"  # the link entity grew with the link
+    assert bold.offset != b.offset  # the originals were not modified
+
+
+def test_a_link_typed_inside_another_word_or_url_is_not_a_post_link():
+    pl = links({9: 99})
+    text = "x https://example.com/?u=t.me/One_Piece_inEnglish/9 and email@t.me/One_Piece_inEnglish/9"
+    assert pl.remap_text(text, [])[0] == text
+
+
+def test_hyperlinks_are_changed_without_touching_their_text():
+    text = "Click here and there"
+    ents = [
+        types.MessageEntityTextUrl(0, 10, "https://t.me/One_Piece_inEnglish/115"),
+        types.MessageEntityTextUrl(11, 3, "https://t.me/another/115"),
+        types.MessageEntityBold(15, 5),
+    ]
+    new_text, new_ents, n_typed, n_hyper = links({115: 2000}).remap_text(text, ents)
+    assert new_text == text and (n_typed, n_hyper) == (0, 1)
+    assert new_ents[0].url == "https://t.me/One_Piece_inEnglish/2000" and new_ents[1].url == "https://t.me/another/115"
+    assert ents[0].url.endswith("/115")  # the message's own entity objects are left alone
+
+
+def test_buttons_with_a_link_to_a_post_are_rebuilt_other_buttons_are_kept_as_they_are():
+    mk = markup([url_btn("Episode 1", "https://t.me/One_Piece_inEnglish/115"), url_btn("Bot", "https://t.me/goku?start=A")])
+    new, n = links({115: 300}).remap_markup(mk)
+    assert n == 1
+    assert [button_url(b) for b in new.rows[0].buttons] == ["https://t.me/One_Piece_inEnglish/300", "https://t.me/goku?start=A"]
+    assert [b.text for b in new.rows[0].buttons] == ["Episode 1", "Bot"]
+    same, n = links({999: 1}).remap_markup(mk)
+    assert n == 0 and same is mk  # nothing to change: the very same keyboard comes back
+    assert links().remap_markup(None) == (None, 0)
+
+
+def test_links_are_counted_even_before_the_new_ids_are_known():
+    text = "https://t.me/One_Piece_inEnglish/5 and https://t.me/other/5"
+    ents = [types.MessageEntityTextUrl(0, 5, "https://t.me/c/1234567/6")]
+    mk = markup([url_btn("x", "https://t.me/One_Piece_inEnglish/7")])
+    assert links().count(text, ents, mk) == 3
+
+
+def test_a_whole_message_is_relinked_or_reported_unchanged():
+    msg = make_msg(
+        5, "Next: https://t.me/One_Piece_inEnglish/115", [types.MessageEntityBold(0, 4)],
+        markup([url_btn("Go", "https://t.me/One_Piece_inEnglish/116")]),
+    )
+    assert relink_message(msg, links({1: 2})) is None
+    rl = relink_message(msg, links({115: 900, 116: 901}))
+    assert rl.text == "Next: https://t.me/One_Piece_inEnglish/900" and (rl.n_typed, rl.n_hyper, rl.n_buttons, rl.links) == (1, 0, 1, 2)
+    assert rl.text_changed and rl.markup_changed
+    assert button_url(rl.markup.rows[0].buttons[0]).endswith("/901")
+    # running it again on the result changes nothing: the new ids are not old ids
+    again = make_msg(5, rl.text, rl.entities, rl.markup)
+    assert relink_message(again, links({115: 900, 116: 901})) is None
+
+
+def test_posts_that_moved_twice_point_at_their_latest_copy():
+    assert resolve_chain({1: 5, 5: 9, 2: 6}) == {1: 9, 5: 9, 2: 6}
+    assert resolve_chain({}) == {}
+
+
+def test_offsets_use_utf16_units():
+    text = "😀😀 https://t.me/One_Piece_inEnglish/3"
+    assert len(add_surrogate(text)) == 4 + 1 + len("https://t.me/One_Piece_inEnglish/3")
+    new_text, _, n, _ = links({3: 30}).remap_text(text, [])
+    assert new_text == "😀😀 https://t.me/One_Piece_inEnglish/30" and n == 1

tests/test_repost_links.py

@@ -0,0 +1,389 @@
+"""/repost keeps what a plain copy would lose: answers (replies), pins, and links to other posts of the channel."""
+import asyncio
+
+from telethon import functions, types
+
+from app.repost_engine import RepostOptions, plan_repost, relink_copies, run_repost
+from app.tgutil import button_url
+
+from .dbutil import db_url_for, fresh_db
+from .fakes import FakeChannelClient, make_msg, markup, photo_media, service_msg, url_btn
+from .test_repost import edit_requests, forwarded_ids, setup
+
+CH = "testch"  # the channel's username in the database (see setup())
+
+
+def build_linked():
+    """Posts that point at each other, answer each other and are pinned."""
+    return {
+        1: make_msg(1, "Intro"),
+        2: make_msg(2, "Next: part five", [types.MessageEntityTextUrl(6, 9, f"https://t.me/{CH}/5")]),  # forward reference
+        3: make_msg(
+            3,
+            f"Index: https://t.me/{CH}/1 and https://t.me/c/1/2 and https://t.me/other_chan/2 end",
+            [types.MessageEntityBold(len("Index: https://t.me/testch/1 and https://t.me/c/1/2 and https://t.me/other_chan/2 "), 3)],
+            markup([url_btn("Go to 4", f"https://t.me/{CH}/4?single")]),  # forward reference in a button
+        ),
+        4: make_msg(4, "Fourth", reply_to=3),
+        5: make_msg(5, "Fifth", [types.MessageEntityTextUrl(0, 5, f"https://t.me/{CH}/1?single")], pinned=True, reply_to=4),
+        6: service_msg(6),
+        7: make_msg(7, "", media=photo_media(71), grouped_id=70, reply_to=5),
+        8: make_msg(8, "album caption", media=photo_media(72), grouped_id=70),
+        9: make_msg(9, f"Ghost https://t.me/{CH}/999", []),  # a post that never existed: left as it is
+        10: make_msg(10, "Last", pinned=True),
+    }
+
+
+def run(coro):
+    return asyncio.run(coro)
+
+
+async def linked_setup(tmp_path, chan=None, **kw):
+    chan = chan or FakeChannelClient(build_linked())
+    kw.setdefault("old", None)
+    kw.setdefault("new", None)
+    return await setup(tmp_path, chan, **kw)
+
+
+def new_posts(chan, above=10):
+    return {i: m for i, m in chan.msgs.items() if i > above and not isinstance(m, types.MessageService)}
+
+
+def reply_of(m):
+    return m.reply_to.reply_to_msg_id if m.reply_to else None
+
+
+def test_the_plan_tells_what_will_be_kept(tmp_path):
+    async def go():
+        db, ch, chan, _ = await linked_setup(tmp_path)
+        plan = await plan_repost(chan, ch, RepostOptions())
+        assert plan.error is None and plan.units == 8
+        assert plan.pinned == 2 and plan.replies == 3
+        assert (plan.post_link_posts, plan.post_links) == (4, 6)  # posts 2, 3 (three links), 5 and 9
+        await db.close()
+
+    run(go())
+
+
+def test_replies_pins_and_links_follow_the_copies(tmp_path):
+    async def go():
+        db, ch, chan, mig = await linked_setup(tmp_path)
+        res = await run_repost(chan, db, ch, mig, 99, delay=0, settle_pause=0)
+        assert (res.copied_units, res.copied_messages, res.failed, res.aborted) == (8, 9, [], None)
+        new = new_posts(chan)
+        assert sorted(new) == list(range(11, 11 + 9 + 0)) or sorted(new) == [11, 12, 13, 14, 15, 16, 17, 18, 19]
+
+        # answers: the copy of 4 answers the copy of 3, 5 answers 4, the album answers 5
+        assert reply_of(new[11]) is None and reply_of(new[12]) is None and reply_of(new[13]) is None
+        assert (reply_of(new[14]), reply_of(new[15]), reply_of(new[16]), reply_of(new[17])) == (13, 14, 15, 15)
+        assert (res.replies, res.reply_lost, res.reply_dropped) == (3, 0, 0)
+
+        # links typed in the text and hyperlinks: backward references were right from the start ...
+        assert new[13].message == f"Index: https://t.me/{CH}/11 and https://t.me/c/1/12 and https://t.me/other_chan/2 end"
+        bold = new[13].entities[0]
+        assert new[13].message.encode("utf-16-le")[bold.offset * 2 : (bold.offset + bold.length) * 2].decode("utf-16-le") == "end"
+        link5 = [e for e in new[15].entities if isinstance(e, types.MessageEntityTextUrl)][0]
+        assert link5.url == f"https://t.me/{CH}/11?single" and new[15].message == "Fifth"
+        # ... forward references were fixed afterwards (post 2 -> 5 is now 15, the button 4 is now 14)
+        link2 = [e for e in new[12].entities if isinstance(e, types.MessageEntityTextUrl)][0]
+        assert link2.url == f"https://t.me/{CH}/15" and (link2.offset, link2.length) == (6, 9)
+        assert button_url(new[13].reply_markup.rows[0].buttons[0]) == f"https://t.me/{CH}/14?single"
+        assert new[13].reply_markup.rows[0].buttons[0].text == "Go to 4"
+        assert new[18].message == f"Ghost https://t.me/{CH}/999"  # no copy of 999: untouched
+        fin = res.final
+        assert (fin.checked, fin.relinked, fin.links, fin.failed) == (9, 2, 2, [])
+        assert len(edit_requests(chan)) == 2  # only the two posts with a forward reference are edited
+
+        # pins: the copies of the pinned posts, oldest first; the originals stay pinned; no notice is left behind
+        assert (fin.pin_wanted, fin.pinned, fin.pin_error) == (2, 2, None)
+        assert chan.pinned_order == [15, 19]
+        assert new[15].pinned and new[19].pinned and not new[11].pinned
+        assert chan.msgs[5].pinned and chan.msgs[10].pinned
+        assert [m.id for m in chan.msgs.values() if isinstance(m, types.MessageService)] == [6]
+
+        # My posts shows exactly what the channel shows
+        p12, p13 = await db.find_by_message(1, 12), await db.find_by_message(1, 13)
+        assert p12.entities[0]["u"] == f"https://t.me/{CH}/15"
+        assert p13.buttons == [[{"t": "Go to 4", "u": f"https://t.me/{CH}/14?single"}]]
+        assert p13.text == new[13].message
+        await db.close()
+
+    run(go())
+
+
+def test_what_needs_no_change_is_still_copied_by_telegram_itself(tmp_path):
+    async def go():
+        db, ch, chan, mig = await linked_setup(tmp_path)
+        await run_repost(chan, db, ch, mig, 99, delay=0, settle_pause=0)
+        # intro, the forward reference (fixed afterwards), the ghost link and the last post: no reply, no buttons
+        assert forwarded_ids(chan) == [[1], [2], [9], [10]]
+        reqs = [r for r in chan.requests if isinstance(r, functions.messages.SendMessageRequest)]
+        assert [r.reply_to.reply_to_msg_id for r in reqs if r.reply_to] == [13, 14]
+        await db.close()
+
+    run(go())
+
+
+def test_running_it_again_changes_nothing(tmp_path):
+    async def go():
+        db, ch, chan, mig = await linked_setup(tmp_path)
+        await run_repost(chan, db, ch, mig, 99, delay=0, settle_pause=0)
+        edits, requests = len(edit_requests(chan)), len(chan.requests)
+        again = await run_repost(chan, db, ch, mig, 99, delay=0, settle_pause=0)
+        assert (again.copied_units, again.skipped_done) == (0, 8)
+        assert (again.final.relinked, again.final.pin_wanted, again.final.pinned) == (0, 0, 0)
+        assert len(edit_requests(chan)) == edits
+        assert not any(isinstance(r, functions.messages.UpdatePinnedMessageRequest) for r in chan.requests[requests:])
+        await db.close()
+
+    run(go())
+
+
+def test_a_channel_that_forbids_forwarding_gets_the_same_result(tmp_path):
+    async def go():
+        chan = FakeChannelClient(build_linked())
+        chan.restrict_forwards = True
+        db, ch, chan, mig = await linked_setup(tmp_path, chan)
+        res = await run_repost(chan, db, ch, mig, 99, delay=0, settle_pause=0)
+        assert (res.failed, res.aborted, res.forwarded, res.rebuilt) == ([], None, 0, 8)
+        new = new_posts(chan)
+        assert (reply_of(new[14]), reply_of(new[15]), reply_of(new[16])) == (13, 14, 15)
+        assert chan.pinned_order == [15, 19]
+        assert res.final.relinked == 2
+        await db.close()
+
+    run(go())
+
+
+def test_a_post_waits_for_the_post_it_answers(tmp_path):
+    """The parent's copy fails (Telegram never shows its buttons): the child is not copied as a loose post."""
+
+    async def go():
+        chan = FakeChannelClient(
+            {
+                1: make_msg(1, "Parent", markup=markup([url_btn("Get", "https://t.me/goku?start=P")])),
+                2: make_msg(2, "Child", reply_to=1),
+                3: make_msg(3, "Other"),
+            }
+        )
+        chan.drop_all_markup = True
+        db, ch, chan, mig = await linked_setup(tmp_path, chan)
+        res = await run_repost(chan, db, ch, mig, 99, delay=0, settle_pause=0)
+        assert [n for _, n in res.failed] == ["ButtonsNotShown", "ReplyParentMissing"]
+        assert res.copied_units == 1 and res.final.relinked == 0  # only post 3 was copied
+        assert [m.message for m in new_posts(chan, 3).values()] == ["Other"]
+        chan.drop_all_markup = False  # Telegram behaves again: "Try the rest again"
+        res2 = await run_repost(chan, db, ch, mig, 99, delay=0, settle_pause=0)
+        assert (res2.copied_units, res2.failed) == (2, [])
+        new = new_posts(chan, 3)
+        parent, child = [m for m in new.values() if m.message == "Parent"][0], [m for m in new.values() if m.message == "Child"][0]
+        assert reply_of(child) == parent.id
+        await db.close()
+
+    run(go())
+
+
+def test_a_reply_to_a_post_outside_a_trial_run_is_copied_without_the_reply(tmp_path):
+    async def go():
+        chan = FakeChannelClient({1: make_msg(1, "Parent"), 2: make_msg(2, "Child", reply_to=1), 3: make_msg(3, "Gone parent", reply_to=99)})
+        db, ch, chan, mig = await linked_setup(tmp_path, chan, first_id=2, partial=True)
+        res = await run_repost(chan, db, ch, mig, 99, delay=0, settle_pause=0)
+        assert (res.copied_units, res.failed, res.replies, res.reply_dropped) == (2, [], 0, 2)
+        assert all(reply_of(m) is None for m in new_posts(chan, 3).values())
+        await db.close()
+
+    run(go())
+
+
+def test_media_that_cannot_be_rebuilt_is_copied_without_its_reply(tmp_path):
+    async def go():
+        chan = FakeChannelClient({1: make_msg(1, "Parent"), 2: make_msg(2, "Odd", media=types.MessageMediaUnsupported(), reply_to=1)})
+        db, ch, chan, mig = await linked_setup(tmp_path, chan)
+        res = await run_repost(chan, db, ch, mig, 99, delay=0, settle_pause=0)
+        assert (res.copied_units, res.failed, res.reply_lost) == (2, [], 1)
+        assert forwarded_ids(chan) == [[1], [2]]
+        await db.close()
+
+    run(go())
+
+
+def test_pinning_needs_a_right_the_bot_may_lack(tmp_path):
+    async def go():
+        chan = FakeChannelClient(build_linked())
+        chan.rights = dict(admin=True, post=True, edit=False, delete=True, invite=False, add_admins=False)
+        db, ch, chan, mig = await linked_setup(tmp_path, chan)
+        res = await run_repost(chan, db, ch, mig, 99, delay=0, settle_pause=0)
+        assert (res.failed, res.aborted, res.copied_units) == ([], None, 8)  # the copies are fine
+        assert (res.final.pin_wanted, res.final.pinned, res.final.pin_error) == (2, 0, "ChatAdminRequiredError")
+        assert chan.pinned_order == []
+        await db.close()
+
+    run(go())
+
+
+def test_links_in_copies_made_earlier_can_be_fixed_afterwards(tmp_path):
+    """A finished repost whose originals are gone: the copies still point at the old ids (what /fixlinks repairs)."""
+
+    async def go():
+        db = await fresh_db(db_url_for(tmp_path))
+        await db.save_channel(1, 5, "Test", CH, 99)
+        ch = await db.get_channel(1)
+        text = f"Part 1 https://t.me/{CH}/3 and https://t.me/{CH}/4 and https://t.me/{CH}/50"
+        chan = FakeChannelClient(
+            {
+                11: make_msg(11, "one", out=True),
+                12: make_msg(12, text, out=True, markup=markup([url_btn("Two", f"https://t.me/{CH}/1")])),
+                13: make_msg(13, "three", out=True),
+                14: make_msg(14, "four", out=True),
+            }
+        )
+        ids = {1: 11, 2: 12, 3: 13, 4: 14}
+        await db.create_post(channel_id=1, message_id=12, status="sent", source="bot", text=text, entities=[], buttons=[[{"t": "Two", "u": f"https://t.me/{CH}/1"}]])
+        dry = await relink_copies(chan, db, ch, ids, [11, 12, 13, 14], dry_run=True)
+        assert (dry.checked, dry.relinked, dry.links) == (4, 1, 3) and chan.edits == []
+        res = await relink_copies(chan, db, ch, ids, [11, 12, 13, 14], delay=0, settle_pause=0)
+        assert (res.relinked, res.links, res.failed) == (1, 3, [])
+        assert chan.msgs[12].message == f"Part 1 https://t.me/{CH}/13 and https://t.me/{CH}/14 and https://t.me/{CH}/50"
+        assert button_url(chan.msgs[12].reply_markup.rows[0].buttons[0]) == f"https://t.me/{CH}/11"
+        saved = await db.find_by_message(1, 12)
+        assert saved.text == chan.msgs[12].message and saved.buttons[0][0]["u"].endswith("/11")
+        # nothing left to do the second time
+        again = await relink_copies(chan, db, ch, ids, [11, 12, 13, 14], delay=0, settle_pause=0)
+        assert (again.relinked, again.links) == (0, 0)
+        # a copy that was deleted in the meantime is reported, not an error
+        del chan.msgs[14]
+        gone = await relink_copies(chan, db, ch, ids, [11, 12, 13, 14], delay=0, settle_pause=0)
+        assert gone.missing == 1
+        await db.close()
+
+    run(go())
+
+
+def test_the_copies_of_every_repost_are_listed_oldest_first_and_undone_ones_left_out(tmp_path):
+    async def go():
+        db, ch, chan, mig = await linked_setup(tmp_path)
+        await run_repost(chan, db, ch, mig, 99, delay=0, settle_pause=0)
+        pairs = await db.migration_pairs(channel_id=1)
+        assert pairs == [(1, 11), (2, 12), (3, 13), (4, 14), (5, 15), (7, 16), (8, 17), (9, 18), (10, 19)]
+        assert await db.migration_pairs(mid="m1") == pairs and await db.migration_pairs(mid="nope") == []
+        await db.mark_migration_deleted("m1", 1, "new", [18, 19])
+        assert await db.migration_pairs(channel_id=1) == pairs[:-2]
+        assert len(await db.migration_pairs(channel_id=1, alive_only=False)) == 9
+        await db.close()
+
+    run(go())
+
+
+# ------------------------------------------------------------------------------ as the owner sees it
+def test_the_plan_and_the_result_message_mention_replies_links_and_pins(tmp_path):
+    from .harness import norm
+    from .test_repost_flow import make_app, pick
+
+    async def go():
+        app = await make_app(tmp_path, build_linked())
+        await pick(app)
+        plan = norm(app.out.last_text)
+        assert "6 link(s) in 4 post(s) point at other posts of this channel" in plan
+        assert "3 post(s) answer another post" in plan and "2 pinned post(s)" in plan
+        assert "pinned state" not in plan  # it is kept now
+        await app.press(app.out.callback_data("Start copying"))
+        done = norm(app.out.last_text)
+        assert "all posts copied in order" in done
+        assert "3 post(s) answer the copy of the post they answered" in done
+        assert "2 link(s) in 2 post(s) now point at the new copies" in done
+        assert "2 pinned post(s) are pinned again" in done
+        assert app.tg.pinned_order == [15, 19]
+        await app.db.close()
+
+    asyncio.run(go())
+
+
+def test_a_missing_pin_right_is_explained_in_the_result(tmp_path):
+    from .harness import norm
+    from .test_repost_flow import make_app, pick
+
+    async def go():
+        app = await make_app(tmp_path, build_linked())
+        app.tg.rights = dict(admin=True, post=True, edit=False, delete=True, invite=False, add_admins=False)
+        await pick(app)
+        await app.press(app.out.callback_data("Start copying"))
+        done = norm(app.out.last_text)
+        assert "2 copy/copies could not be pinned (ChatAdminRequiredError)" in done and "Edit messages of others" in done
+        assert "all posts copied in order" in done  # the copy itself is complete
+        await app.db.close()
+
+    asyncio.run(go())
+
+
+async def finished_repost(app, deleted_originals=True):
+    """A repost that is over: the copies (ids 11-13) still point at the old ids 1-3, the originals are gone."""
+    text = f"Part one: https://t.me/{CH}/3 and https://t.me/{CH}/2"
+    app.tg.msgs = {
+        11: make_msg(11, "one", out=True),
+        12: make_msg(12, text, out=True, markup=markup([url_btn("Back", f"https://t.me/{CH}/1")])),
+        13: make_msg(13, "three", out=True),
+    }
+    app.tg._top = 13
+    await app.db.create_migration("m9", 1, old=None, new=None, include_typed=True, include_posts=False, first_id=1, last_id=3, partial=False, user_id=1)
+    rows = [{"old_id": o, "new_id": n, "post": dict(text="", entities=[], buttons=[], media_kind=None, media_file_id=None, link_preview=False)} for o, n in ((1, 11), (2, 12), (3, 13))]
+    await app.db.record_repost("m9", 1, rows, 1)
+    if deleted_originals:
+        await app.db.mark_migration_deleted("m9", 1, "old", [1, 2, 3])
+    return text
+
+
+def test_fixlinks_repairs_a_repost_that_was_made_without_it(tmp_path):
+    from .harness import norm
+    from .test_repost_flow import make_app
+
+    async def go():
+        app = await make_app(tmp_path, {1: make_msg(1, "x")})
+        await finished_repost(app)
+        await app.text("/fixlinks")
+        assert "Fix links to reposted posts" in norm(app.out.last_text) and "Which channel" in norm(app.out.last_text)
+        await app.press(app.out.callback_data("Test"))
+        scan = norm(app.out.last_text)
+        assert "3 link(s) in 1 post(s) still point at posts that were reposted and deleted" in scan
+        assert app.tg.edits == [] or all(e[0] != "raw" for e in app.tg.edits)  # looking changes nothing
+        assert app.tg.msgs[12].message.endswith("/3 and https://t.me/testch/2")
+        await app.press(app.out.callback_data("Fix 3 link"))
+        assert "3 link(s) in 1 post(s) now point at the copies" in norm(app.out.last_text)
+        assert app.tg.msgs[12].message == f"Part one: https://t.me/{CH}/13 and https://t.me/{CH}/12"
+        assert button_url(app.tg.msgs[12].reply_markup.rows[0].buttons[0]) == f"https://t.me/{CH}/11"
+        # a second run finds nothing
+        await app.text("/fixlinks")
+        await app.press(app.out.callback_data("Test"))
+        assert "no link points at a deleted original" in norm(app.out.last_text)
+        await app.db.close()
+
+    asyncio.run(go())
+
+
+def test_fixlinks_leaves_a_repost_alone_while_its_originals_still_exist(tmp_path):
+    from .harness import norm
+    from .test_repost_flow import make_app
+
+    async def go():
+        app = await make_app(tmp_path, {1: make_msg(1, "x")})
+        await finished_repost(app, deleted_originals=False)
+        await app.text("/fixlinks")
+        await app.press(app.out.callback_data("Test"))
+        assert "no repost with deleted originals is recorded" in norm(app.out.last_text)
+        await app.db.close()
+
+    asyncio.run(go())
+
+
+def test_fixlinks_is_for_the_owner_only(tmp_path):
+    from .test_repost_flow import make_app
+
+    async def go():
+        app = await make_app(tmp_path, {1: make_msg(1, "x")}, admins=frozenset({7}))
+        app.uid = 7
+        before = len(app.out.log)
+        await app.text("/fixlinks")
+        assert "private" in str(app.out.log[before:][-1][1]).lower() or len(app.out.log) == before + 1
+        assert "Which channel" not in str(app.out.last_text)
+        await app.db.close()
+
+    asyncio.run(go())

tests/test_shift.py

@@ -0,0 +1,436 @@
+"""/shift: posts of one channel copied into another - range, buttons, replies, links between the posts, undo."""
+import asyncio
+from types import SimpleNamespace
+
+import pytest
+from telethon import functions, types
+
+from app.channelref import classify_ref, resolve_channel
+from app.common import UserError
+from app.handlers.shift import parse_shift_args
+from app.repost_engine import RepostOptions, plan_repost
+from app.shift_engine import delete_shift_copies, run_shift
+from app.tgutil import button_url
+
+from .dbutil import db_url_for, fresh_db
+from .fakes import FakeChannelClient, cb_btn, make_msg, markup, photo_media, service_msg, url_btn
+from .harness import App, norm
+from .test_repost_flow import fill
+
+SRC = SimpleNamespace(id=1, access_hash=5, title="Anime Source", username="srcch")
+
+
+def source_posts():
+    return {
+        1: make_msg(1, "Welcome", [types.MessageEntityBold(0, 7)]),
+        2: make_msg(
+            2, "Click below",
+            markup=markup([url_btn("Download", "https://t.me/goku?start=AAA")], [cb_btn("👍 3", b"r")]),
+        ),
+        3: make_msg(3, "Caption with link", [types.MessageEntityTextUrl(13, 4, "https://t.me/srcch/5")], media=photo_media(31)),
+        4: service_msg(4),
+        5: make_msg(5, "Fifth", reply_to=2, pinned=True),
+        6: make_msg(6, "", media=photo_media(61), grouped_id=60),
+        7: make_msg(7, "album caption", media=photo_media(62), grouped_id=60),
+        8: make_msg(8, "See https://t.me/srcch/1 and https://t.me/c/1/2 and https://t.me/elsewhere/9"),
+    }
+
+
+def old_posts():
+    """Posts the destination already has, so that the copies get ids that differ from the originals'."""
+    return {i: make_msg(i, f"existing {i}", out=True) for i in range(1, 11)}
+
+
+def two_channels(src_msgs=None, dst_msgs=None):
+    src = FakeChannelClient(source_posts() if src_msgs is None else src_msgs, username="srcch", title="Anime Source")
+    dst = src.add_channel(FakeChannelClient(old_posts() if dst_msgs is None else dst_msgs, username="dstch", cid=2, title="Anime Backup"))
+    return src, dst
+
+
+async def engine_setup(tmp_path, src, first=1, last=8):
+    db = await fresh_db(db_url_for(tmp_path))
+    await db.save_channel(2, 6, "Anime Backup", "dstch", 1)
+    dst_row = await db.get_channel(2)
+    await db.create_shift("s1", src=SRC, dst_channel_id=2, first_id=first, last_id=last, user_id=1)
+    return db, dst_row, await db.get_shift("s1")
+
+
+def reply_of(m):
+    return m.reply_to.reply_to_msg_id if m.reply_to else None
+
+
+def run(coro):
+    return asyncio.run(coro)
+
+
+# ------------------------------------------------------------------------------------- the command line
+def test_arguments_are_read_as_source_destination_and_an_optional_range():
+    assert parse_shift_args("@a @b") == ("@a", "@b", None, None)
+    assert parse_shift_args("@a @b 15 20") == ("@a", "@b", 15, 20)
+    assert parse_shift_args("@a @b 20 15") == ("@a", "@b", 15, 20)  # either order
+    assert parse_shift_args("@a @b 15") == ("@a", "@b", 15, 15)  # one id = just that post
+    assert parse_shift_args("https://t.me/+AbCdEfGhIj -1001234567890") == ("https://t.me/+AbCdEfGhIj", "-1001234567890", None, None)
+    for bad in ("", "@a", "@a @b x", "@a @b 0", "@a @b 1 2 3", "@a @b -5"):
+        with pytest.raises(ValueError):
+            parse_shift_args(bad)
+
+
+def test_every_kind_of_channel_reference_is_told_apart():
+    assert classify_ref("@srcch") == ("username", "srcch")
+    assert classify_ref("https://t.me/srcch") == ("username", "srcch")
+    assert classify_ref("https://t.me/srcch/15") == ("username", "srcch")
+    assert classify_ref("-1001234567890") == ("id", 1234567890)
+    assert classify_ref("https://t.me/c/1234567890/5") == ("id", 1234567890)
+    assert classify_ref("https://t.me/+AbCdEfGhIjKl") == ("invite", "AbCdEfGhIjKl")
+    assert classify_ref("t.me/joinchat/AbCdEfGhIjKl") == ("invite", "AbCdEfGhIjKl")
+    assert classify_ref("tg://join?invite=AbCdEfGhIjKl") == ("invite", "AbCdEfGhIjKl")
+    assert classify_ref("???") == (None, None)
+
+
+def test_channels_are_found_by_name_id_or_invite_link(tmp_path):
+    async def go():
+        app = App(tmp_path)
+        await app.start()
+        src, dst = two_channels()
+        app.tg.registry = src.registry
+        app.tg.registry[1] = app.tg
+        app.tg.username, app.tg.title = "srcch", "Anime Source"
+        app.tg.add_channel(dst)
+        await app.db.save_channel(1, 5, "Anime Source", "srcch", 1)  # registered
+        # registered: found in the database, whatever the spelling
+        for text in ("@srcch", "SRCCH", "https://t.me/srcch/12", "1"):
+            ref = await resolve_channel(app.ctx, text)
+            assert (ref.id, ref.title, ref.registered) == (1, "Anime Source", True), text
+        # not registered: asked from Telegram by name, by id (a channel the bot has met) or by invite link
+        ref = await resolve_channel(app.ctx, "@dstch")
+        assert (ref.id, ref.title, ref.username, ref.registered) == (2, "Anime Backup", "dstch", False)
+        assert (await resolve_channel(app.ctx, "2")).id == 2
+        dst.invite_hash = "AbCdEfGhIjKl"
+        assert (await resolve_channel(app.ctx, "https://t.me/+AbCdEfGhIjKl")).id == 2
+        # things that cannot be used say why
+        for text, part in (
+            ("???", "not a channel I understand"),
+            ("@nosuchchannel", "couldn't find a channel"),
+            ("99", "don't know a channel with the id 99"),
+            ("https://t.me/+ZzZzZzZzZz", "invite link doesn't work"),
+        ):
+            with pytest.raises(UserError, match=part):
+                await resolve_channel(app.ctx, text)
+        await app.db.close()
+
+    run(go())
+
+
+def test_a_group_or_a_user_is_not_a_channel(tmp_path):
+    async def go():
+        app = App(tmp_path)
+        await app.start()
+        group = FakeChannelClient({}, username="agroup", cid=3)
+        group._channel_obj = lambda: types.Channel(id=3, title="G", photo=types.ChatPhotoEmpty(), date=None, access_hash=1, username="agroup", megagroup=True)
+        app.tg.add_channel(group)
+        with pytest.raises(UserError, match="group, not a channel"):
+            await resolve_channel(app.ctx, "@agroup")
+        await app.db.close()
+
+    run(go())
+
+
+# ------------------------------------------------------------------------------------------ the engine
+def test_posts_arrive_in_the_other_channel_with_everything_they_had(tmp_path):
+    async def go():
+        src, dst = two_channels()
+        db, dst_row, shift = await engine_setup(tmp_path, src)
+        res = await run_shift(src, db, shift, dst_row, 99, delay=0, settle_pause=0)
+        assert (res.copied_units, res.copied_messages, res.failed, res.aborted) == (6, 7, [], None)
+        new = {i: m for i, m in dst.msgs.items() if i > 10}
+        assert list(new) == [11, 12, 13, 14, 15, 16, 17]  # same order as the originals
+
+        assert new[11].message == "Welcome" and isinstance(new[11].entities[0], types.MessageEntityBold)
+        # the button post: link button kept, the other bot's 👍 button is not copied (it would be dead)
+        rows = new[12].reply_markup.rows
+        assert len(rows) == 1 and button_url(rows[0].buttons[0]) == "https://t.me/goku?start=AAA" and rows[0].buttons[0].text == "Download"
+        assert new[13].media.photo.id == 31
+        # a reply answers the COPY of the post it answered; a pin is not copied
+        assert reply_of(new[14]) == 12 and not new[14].pinned
+        assert new[15].grouped_id and new[15].grouped_id == new[16].grouped_id and new[16].message == "album caption"
+        # links to other source posts follow the content: backward ones at once, forward ones afterwards
+        assert new[17].message == "See https://t.me/dstch/11 and https://t.me/dstch/12 and https://t.me/elsewhere/9"
+        link = [e for e in new[13].entities if isinstance(e, types.MessageEntityTextUrl)][0]
+        assert link.url == "https://t.me/dstch/14" and (link.offset, link.length) == (13, 4)
+        assert (res.final.relinked, res.final.links) == (1, 1)
+
+        # the source was only read
+        assert sorted(src.msgs) == [1, 2, 3, 4, 5, 6, 7, 8] and src.msgs[5].pinned
+        assert src.delete_requests == [] and src.edits == []
+        assert not any(isinstance(r, (functions.messages.SendMessageRequest, functions.messages.SendMediaRequest)) for r in src.requests)
+
+        # recorded for My posts: only the destination has posts, exactly as they look there
+        assert await db.shift_pairs("s1") == [(1, 11), (2, 12), (3, 13), (5, 14), (6, 15), (7, 16), (8, 17)]
+        assert await db.sent_posts(1) == []
+        mine = {p.message_id: p for p in await db.sent_posts(2)}
+        assert sorted(mine) == [11, 12, 13, 14, 15, 16, 17] and mine[12].source == "bot"
+        assert mine[12].buttons == [[{"t": "Download", "u": "https://t.me/goku?start=AAA"}]]
+        assert mine[13].entities[0]["u"] == "https://t.me/dstch/14"
+        assert mine[17].text == new[17].message and mine[13].media_kind == "photo"
+        await db.close()
+
+    run(go())
+
+
+def test_only_the_posts_in_the_range_are_copied(tmp_path):
+    async def go():
+        src, dst = two_channels()
+        db, dst_row, shift = await engine_setup(tmp_path, src, first=2, last=3)
+        plan = await plan_repost(src, SRC, RepostOptions(), first_id=2, last_id=3)
+        assert (plan.first, plan.last, plan.units, plan.partial) == (2, 3, 2, True)
+        res = await run_shift(src, db, shift, dst_row, 99, delay=0, settle_pause=0)
+        assert (res.copied_units, res.failed) == (2, [])
+        assert [m.message for i, m in sorted(dst.msgs.items()) if i > 10] == ["Click below", "Caption with link"]
+        await db.close()
+
+    run(go())
+
+
+def test_a_reply_to_a_post_outside_the_range_is_copied_without_the_reply(tmp_path):
+    async def go():
+        src, dst = two_channels()
+        db, dst_row, shift = await engine_setup(tmp_path, src, first=5, last=5)
+        res = await run_shift(src, db, shift, dst_row, 99, delay=0, settle_pause=0)
+        assert (res.copied_units, res.replies, res.reply_dropped) == (1, 0, 1)
+        assert reply_of(dst.msgs[11]) is None
+        await db.close()
+
+    run(go())
+
+
+def test_a_source_that_forbids_saving_is_rebuilt_from_its_parts(tmp_path):
+    async def go():
+        src, dst = two_channels()
+        src.restrict_forwards = True
+        db, dst_row, shift = await engine_setup(tmp_path, src)
+        res = await run_shift(src, db, shift, dst_row, 99, delay=0, settle_pause=0)
+        assert (res.failed, res.forwarded, res.rebuilt) == ([], 0, 6)
+        new = {i: m for i, m in dst.msgs.items() if i > 10}
+        assert new[15].grouped_id == new[16].grouped_id and new[13].media.photo.id == 31
+        assert reply_of(new[14]) == 12
+        await db.close()
+
+    run(go())
+
+
+def test_a_stopped_shift_continues_without_copying_anything_twice(tmp_path):
+    async def go():
+        src, dst = two_channels()
+        db, dst_row, shift = await engine_setup(tmp_path, src)
+        calls = []
+
+        async def prog(res):
+            calls.append(res.copied_units)
+
+        res = await run_shift(src, db, shift, dst_row, 99, delay=0, settle_pause=0, progress=prog, should_stop=lambda: len(calls) >= 2)
+        assert res.stopped and res.copied_units == 2 and res.final is None
+        again = await run_shift(src, db, shift, dst_row, 99, delay=0, settle_pause=0)
+        assert (again.copied_units, again.skipped_done, again.failed) == (4, 2, [])
+        assert sorted(i for i in dst.msgs if i > 10) == [11, 12, 13, 14, 15, 16, 17]  # no duplicates
+        # the reply of the 3rd copy found the copy made in the first run
+        assert reply_of(dst.msgs[14]) == 12
+        await db.close()
+
+    run(go())
+
+
+def test_undo_removes_only_the_copies(tmp_path):
+    async def go():
+        src, dst = two_channels()
+        db, dst_row, shift = await engine_setup(tmp_path, src)
+        await run_shift(src, db, shift, dst_row, 99, delay=0, settle_pause=0)
+        res = await delete_shift_copies(src, db, dst_row, shift, delay=0)
+        assert (res.deleted, res.remaining, res.error) == (7, 0, None)
+        assert sorted(dst.msgs) == list(range(1, 11)) and sorted(src.msgs) == [1, 2, 3, 4, 5, 6, 7, 8]
+        assert await db.sent_posts(2) == [] and await db.shift_pairs("s1") == []
+        assert (await db.shift_counts("s1"))["new_left"] == 0
+        await db.close()
+
+    run(go())
+
+
+def test_a_post_that_was_shifted_before_is_known(tmp_path):
+    async def go():
+        src, dst = two_channels()
+        db, dst_row, shift = await engine_setup(tmp_path, src)
+        await run_shift(src, db, shift, dst_row, 99, delay=0, settle_pause=0)
+        assert await db.shifted_ids(1, 2, 1, 5) == {1, 2, 3, 5}
+        assert await db.shifted_ids(1, 3, 1, 5) == set() and await db.shifted_ids(2, 1, 1, 5) == set()
+        await db.mark_shift_deleted("s1", 2, [11])
+        assert await db.shifted_ids(1, 2, 1, 5) == {2, 3, 5}
+        await db.close()
+
+    run(go())
+
+
+# ---------------------------------------------------------------------------- as the owner experiences it
+async def shift_app(tmp_path, **cfg):
+    cfg.setdefault("edit_delay", 0.0)
+    app = App(tmp_path, **cfg)
+    await app.start()
+    app.tg.username, app.tg.title = "srcch", "Anime Source"
+    fill(app.tg, source_posts())
+    dst = app.tg.add_channel(FakeChannelClient(old_posts(), username="dstch", cid=2, title="Anime Backup"))
+    await app.db.save_channel(1, 5, "Anime Source", "srcch", 1)
+    await app.db.save_channel(2, 6, "Anime Backup", "dstch", 1)
+    return app, dst
+
+
+def test_usage_and_bad_commands_are_explained(tmp_path):
+    async def go():
+        app, _ = await shift_app(tmp_path)
+        await app.text("/shift")
+        text = norm(app.out.last_text)
+        assert "Usage: /shift <source> <destination> [from_id] [to_id]" in text and "15 20" in text
+        for args, part in (
+            ("@srcch", "Give the source and the destination"),
+            ("@srcch @dstch x", "“x” is not a message id"),
+            ("@srcch @srcch", "same channel"),
+            ("@srcch @nosuch", "couldn't find a channel"),
+        ):
+            await app.text(f"/shift {args}")
+            assert part in norm(app.out.last_text), (args, norm(app.out.last_text))
+        assert not any(isinstance(r, functions.messages.SendMessageRequest) for r in app.tg.requests)
+        await app.db.close()
+
+    run(go())
+
+
+def test_the_plan_is_shown_before_anything_is_copied(tmp_path):
+    async def go():
+        app, dst = await shift_app(tmp_path)
+        await app.text("/shift @srcch @dstch 2 5")
+        text = norm(app.out.last_text)
+        assert "Shift plan" in text and "From Anime Source → Anime Backup" in text
+        assert "Post ids 2-5" in text and "3 posts to copy" in text  # 2, 3 and 5 (4 is a service message)
+        assert "1 service message(s)" in text and "1 post(s) answer another post" in text
+        assert "1 link(s) in 1 post(s) point at other posts of Anime Source" in text
+        assert "Nothing is changed in Anime Source" in text
+        assert dst.requests == [] or not any(isinstance(r, functions.messages.SendMessageRequest) for r in dst.requests)
+        assert sorted(dst.msgs) == list(range(1, 11))
+        await app.db.close()
+
+    run(go())
+
+
+def test_cancel_and_an_old_button(tmp_path):
+    async def go():
+        app, dst = await shift_app(tmp_path)
+        await app.text("/shift @srcch @dstch")
+        start = app.out.callback_data("Start copying")
+        await app.press(app.out.callback_data("Cancel"))
+        assert "Cancelled" in norm(app.out.last_text)
+        await app.press(start)
+        assert "expired" in norm(app.out.last_text)
+        assert sorted(dst.msgs) == list(range(1, 11))
+        await app.db.close()
+
+    run(go())
+
+
+def test_the_whole_flow_copy_then_undo(tmp_path):
+    async def go():
+        app, dst = await shift_app(tmp_path)
+        await app.text("/shift @srcch @dstch")
+        await app.press(app.out.callback_data("Start copying"))
+        done = norm(app.out.last_text)
+        assert "Anime Source → Anime Backup: 7 message(s) copied" in done and "registered in My posts" in done
+        assert "The source was not changed" in done and "1 link(s) in 1 post(s) now point at the new copies" in done
+        assert sorted(i for i in dst.msgs if i > 10) == [11, 12, 13, 14, 15, 16, 17]
+        assert len(await app.db.sent_posts(2)) == 7
+
+        await app.press(app.out.callback_data("Undo"))
+        assert "Remove the 7 copied message(s) from Anime Backup" in norm(app.out.last_text)
+        await app.press(app.out.callback_data("Yes, remove"))
+        assert "7 copied message(s) removed from Anime Backup" in norm(app.out.last_text)
+        assert sorted(dst.msgs) == list(range(1, 11)) and await app.db.sent_posts(2) == []
+        sh = await app.db.open_shift()
+        assert sh is None  # nothing is left open
+        await app.db.close()
+
+    run(go())
+
+
+def test_a_destination_the_bot_does_not_know_yet_becomes_one_of_its_channels(tmp_path):
+    async def go():
+        app, dst = await shift_app(tmp_path)
+        await app.db.set_channel_active(2, False)
+        await app.text("/shift @srcch @dstch 1 1")
+        await app.press(app.out.callback_data("Start copying"))
+        assert "1 message(s) copied" in norm(app.out.last_text)
+        assert [c.id for c in await app.db.list_channels()] == [2, 1] or {c.id for c in await app.db.list_channels()} == {1, 2}
+        await app.db.close()
+
+    run(go())
+
+
+def test_the_bot_must_be_allowed_to_post_in_the_destination(tmp_path):
+    async def go():
+        app, dst = await shift_app(tmp_path)
+        dst.rights = dict(admin=True, post=False, edit=True, delete=True, invite=False, add_admins=False)
+        await app.text("/shift @srcch @dstch")
+        assert "needs the “Post messages” admin right in Anime Backup" in norm(app.out.last_text)
+        await app.db.close()
+
+    run(go())
+
+
+def test_an_unreadable_source_is_explained(tmp_path):
+    async def go():
+        app, dst = await shift_app(tmp_path)
+        app.tg.fail["GetMessagesRequest"] = __import__("telethon").errors.ChannelPrivateError(request=None)
+        await app.text("/shift @srcch @dstch")
+        text = norm(app.out.last_text)
+        assert "I can't read Anime Source" in text and "member of the source channel" in text
+        await app.db.close()
+
+    run(go())
+
+
+def test_an_unfinished_shift_must_be_dealt_with_first(tmp_path):
+    async def go():
+        app, dst = await shift_app(tmp_path)
+        await app.db.create_shift("old1", src=SRC, dst_channel_id=2, first_id=1, last_id=8, user_id=1)
+        await app.db.set_shift_status("old1", "stopped")
+        await app.text("/shift @srcch @dstch")
+        text = norm(app.out.last_text)
+        assert "unfinished shift between these two channels" in text and "stopped" in text
+        assert "Continue" in " ".join(t for row in app.out.last_buttons() for t in row)
+        await app.text("/shift")  # without arguments it shows the same unfinished shift
+        assert "Unfinished shift" in norm(app.out.last_text)
+        # Continue copies what is left; the shift is over afterwards
+        await app.press(app.out.callback_data("Continue"))
+        assert "7 message(s) copied" in norm(app.out.last_text)
+        assert await app.db.open_shift() is None
+        await app.db.close()
+
+    run(go())
+
+
+def test_copying_the_same_posts_twice_is_pointed_out(tmp_path):
+    async def go():
+        app, dst = await shift_app(tmp_path)
+        await app.text("/shift @srcch @dstch 1 2")
+        await app.press(app.out.callback_data("Start copying"))
+        await app.text("/shift @srcch @dstch 1 3")
+        assert "2 of these posts were already shifted into Anime Backup before" in norm(app.out.last_text)
+        await app.db.close()
+
+    run(go())
+
+
+def test_shift_is_for_the_owner_only(tmp_path):
+    async def go():
+        app, dst = await shift_app(tmp_path, admins=frozenset({7}))
+        app.uid = 7
+        await app.text("/shift @srcch @dstch")
+        assert "Shift plan" not in norm(app.out.last_text or "")
+        assert sorted(dst.msgs) == list(range(1, 11))
+        await app.db.close()
+
+    run(go())

tests/test_startup.py

@@ -101,7 +101,7 @@ def test_normal_start_up(tmp_path, run_main):
     assert tg.calls == [("start", TOKEN), ("run", None)]
     assert tg.menu == [(c, d) for c, d in bot.COMMANDS]
     names = [c for c, _ in tg.menu]
-    assert "repost" in names and "userbot" in names and "testerror" in names
+    assert "repost" in names and "userbot" in names and "testerror" not in names
     assert "testedit" not in names and "selftest" not in names
     assert tg.handlers  # every module registered its handlers
     assert tg.sent == []  # nothing needed reporting

tests/test_sync.py

@@ -0,0 +1,677 @@
+"""The sync engine: My posts follows what channel admins do outside the bot (deleted, edited and new posts, service messages)."""
+import asyncio
+import datetime
+
+from telethon import errors, types
+
+from app.db import as_utc, utcnow
+from app.servicemsg import remove
+from app.sync_engine import (
+    SyncOptions,
+    diff_post,
+    find_top,
+    live_fields,
+    message_supported,
+    sync_channel,
+    sync_ids,
+)
+from app.tgutil import media_info
+
+from .dbutil import db_url_for, fresh_db
+from .fakes import FakeChannelClient, make_msg, markup, photo_media, service_msg, url_btn, video_media
+
+OLD = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(days=1)
+FAST = dict(pause=0)
+
+
+def old_msg(i, text="", **kw):
+    m = make_msg(i, text, **kw)
+    m.date = OLD
+    return m
+
+
+def foreign(chan, text="", *, age=86400, **kw):
+    """A post somebody else made in the channel (not out)."""
+    m = chan._new(text, kw.pop("entities", None), kw.pop("markup", None), kw.pop("media", None))
+    m.out = False
+    m.date = utcnow() - datetime.timedelta(seconds=age)
+    return m
+
+
+async def saved(db, m, **extra):
+    """The bot's saved copy of live message `m`, exactly as the bot would have saved it a day ago."""
+    fields = dict(
+        channel_id=1, message_id=m.id, status="sent", source="bot", created_by=99, link_preview=False,
+        created_at=OLD, updated_at=OLD, sent_at=OLD, **live_fields(m),
+    )
+    fields.update(extra)
+    return await db.create_post(**fields)
+
+
+async def world(tmp_path, msgs, *, save=True):
+    chan = FakeChannelClient(msgs, username="chan")
+    db = await fresh_db(db_url_for(tmp_path))
+    await db.save_channel(1, 5, "Chan", "chan", 99)
+    ch = await db.get_channel(1)
+    if save:
+        for m in msgs.values():
+            await saved(db, m)
+    return chan, db, ch
+
+
+def run(coro):
+    return asyncio.run(coro)
+
+
+def opts(**kw):
+    return SyncOptions(**{**FAST, "grace": 0, **kw})
+
+
+def three():
+    return {
+        1: old_msg(1, "first"),
+        2: old_msg(2, "second", markup=markup([url_btn("Watch", "https://t.me/a/1")])),
+        3: old_msg(3, "third"),
+    }
+
+
+# ============================================================================================ comparing one post
+def test_a_post_that_is_as_saved_has_no_difference():
+    m = old_msg(1, "hello", markup=markup([url_btn("Go", "https://x.org")]))
+    p = SaveLike(m)
+    d = diff_post(p, m)
+    assert d.fields == {} and not d.buttons_missing and not d.quiet
+
+
+class SaveLike:
+    """Anything with the Post fields."""
+
+    def __init__(self, m, **kw):
+        for k, v in live_fields(m).items():
+            setattr(self, k, v)
+        self.link_preview = False
+        for k, v in kw.items():
+            setattr(self, k, v)
+
+
+def test_text_formatting_and_buttons_that_changed_are_found():
+    m = old_msg(1, "hello world", entities=[types.MessageEntityBold(0, 5)], markup=markup([url_btn("Go", "https://x.org")]))
+    p = SaveLike(m, text="hello", entities=[], buttons=[[{"t": "Go", "u": "https://old.org"}]])
+    d = diff_post(p, m)
+    assert d.fields["text"] == "hello world"
+    assert d.fields["entities"] == [{"k": "bold", "o": 0, "l": 5}]
+    assert d.fields["buttons"] == [[{"t": "Go", "u": "https://x.org"}]]
+    assert not d.quiet and not d.buttons_missing
+
+
+def test_a_keyboard_that_is_not_shown_never_wipes_the_saved_buttons():
+    m = old_msg(1, "hello")  # the channel shows no keyboard
+    p = SaveLike(m, buttons=[[{"t": "Go", "u": "https://x.org"}]])
+    d = diff_post(p, m)
+    assert d.buttons_missing and "buttons" not in d.fields
+
+
+def test_only_spaces_around_the_text_is_a_quiet_correction():
+    m = old_msg(1, "hello")
+    d = diff_post(SaveLike(m, text="hello \n"), m)
+    assert d.fields == {"text": "hello"} and d.quiet
+
+
+def test_media_is_compared_by_its_id_not_by_its_file_reference():
+    m = old_msg(1, "pic", media=photo_media(11))
+    p = SaveLike(m)
+    other_fetch = old_msg(1, "pic", media=photo_media(11))
+    other_fetch.media.photo.file_reference = b"a-fresh-reference"
+    assert diff_post(p, other_fetch).fields == {}  # same photo, new file reference: nothing changed
+    replaced = old_msg(1, "pic", media=photo_media(12))
+    d = diff_post(p, replaced)
+    assert d.fields["media_kind"] == "photo" and d.fields["media_file_id"] == media_info(replaced)[1]
+
+
+def test_media_that_seems_to_have_vanished_is_not_forgotten():
+    with_media = old_msg(1, "pic", media=photo_media(11))
+    p = SaveLike(with_media)
+    stripped = old_msg(1, "pic")  # Telegram showed it without the photo this time
+    assert diff_post(p, stripped).fields == {}
+
+
+def test_what_the_bot_can_hold():
+    assert message_supported(old_msg(1, "text"))
+    assert message_supported(old_msg(1, "", media=photo_media()))
+    assert message_supported(old_msg(1, "", media=video_media()))
+    assert not message_supported(old_msg(1, ""))  # a poll, a sticker ...
+    assert not message_supported(old_msg(1, "  \n "))
+
+
+# ================================================================================================= the first look
+def test_the_first_look_only_notes_where_the_channel_stands(tmp_path):
+    async def go():
+        msgs = three()
+        chan, db, ch = await world(tmp_path, msgs, save=False)  # nothing saved: the channel has history only
+        for i in (4, 5):
+            chan.msgs[i] = old_msg(i, f"older post {i}")
+        chan._top = chan._pts = 5
+        rep = await sync_channel(chan, db, ch, opts())
+        assert rep.first_look and rep.error is None
+        assert rep.adopted == [] and rep.deleted == [] and rep.edited == []  # the history is not imported
+        assert await db.list_posts(1, None, 0, 50) == ([], 0)
+        state = await db.get_sync(1)
+        assert (state.last_top, state.last_pts, state.deferred) == (5, 5, 0)
+        await db.close()
+
+    run(go())
+
+
+def test_a_channel_without_any_message_is_fine(tmp_path):
+    async def go():
+        chan, db, ch = await world(tmp_path, {}, save=False)
+        rep = await sync_channel(chan, db, ch, opts())
+        assert rep.first_look and rep.error is None and rep.adopted == []
+        again = await sync_channel(chan, db, ch, opts())
+        assert again.error is None and again.changed is False
+        await db.close()
+
+    run(go())
+
+
+def test_the_first_look_finds_the_newest_message_below_a_long_run_of_deleted_ids(tmp_path):
+    async def go():
+        chan, db, ch = await world(tmp_path, {1: old_msg(1, "a"), 2: old_msg(2, "b")}, save=False)
+        chan._top, chan._pts = 2, 1000  # a thousand edits and deletes since
+        top = await find_top(chan, types.InputPeerChannel(1, 5), 1000, 0, pause=0)
+        assert top == 2
+        # the newest saved post is a floor: looking stops there
+        assert await find_top(chan, types.InputPeerChannel(1, 5), 1000, 2, pause=0) == 2
+        await db.close()
+
+    run(go())
+
+
+# ============================================================================================= saved posts vs channel
+def test_a_post_deleted_in_the_channel_leaves_my_posts(tmp_path):
+    async def go():
+        chan, db, ch = await world(tmp_path, three())
+        await sync_channel(chan, db, ch, opts())  # the first look
+        del chan.msgs[2]
+        chan._pts += 1
+        rep = await sync_channel(chan, db, ch, opts())
+        assert rep.deleted == [2] and rep.checked == 3
+        assert sorted(p.message_id for p in await db.sent_posts(1)) == [1, 3]
+        await db.close()
+
+    run(go())
+
+
+def test_a_post_edited_in_the_channel_is_updated_here(tmp_path):
+    async def go():
+        chan, db, ch = await world(tmp_path, three())
+        m2 = chan.msgs[2]
+        m2.message = "second, edited by an admin"
+        m2.entities = [types.MessageEntityItalic(0, 6)]
+        m2.reply_markup = markup([url_btn("Watch now", "https://t.me/new/9")])
+        rep = await sync_channel(chan, db, ch, opts())
+        assert rep.edited == [2] and rep.deleted == [] and rep.adopted == []
+        p = await db.find_by_message(1, 2)
+        assert p.text == "second, edited by an admin"
+        assert p.entities == [{"k": "italic", "o": 0, "l": 6}]
+        assert p.buttons == [[{"t": "Watch now", "u": "https://t.me/new/9"}]]
+        again = await sync_channel(chan, db, ch, opts())
+        assert again.edited == [] and again.changed is False  # nothing left to record
+        await db.close()
+
+    run(go())
+
+
+def test_a_missing_keyboard_is_reported_and_the_saved_buttons_stay(tmp_path):
+    async def go():
+        chan, db, ch = await world(tmp_path, three())
+        chan.msgs[2].reply_markup = None  # the channel shows no keyboard any more
+        rep = await sync_channel(chan, db, ch, opts())
+        assert rep.buttons_missing == [2] and rep.edited == []
+        assert (await db.find_by_message(1, 2)).buttons == [[{"t": "Watch", "u": "https://t.me/a/1"}]]
+        await db.close()
+
+    run(go())
+
+
+def test_posts_the_bot_touched_a_moment_ago_are_left_for_later(tmp_path):
+    async def go():
+        chan, db, ch = await world(tmp_path, three())
+        await db.update_post((await db.find_by_message(1, 1)).id, text="just edited through the bot")  # updated_at = now
+        chan.msgs[1].message = "an admin's text"
+        rep = await sync_channel(chan, db, ch, opts(grace=120))
+        assert rep.young == [1] and rep.edited == [] and rep.checked == 2
+        assert (await db.find_by_message(1, 1)).text == "just edited through the bot"
+        await db.close()
+
+    run(go())
+
+
+def test_if_every_saved_post_looks_deleted_nothing_is_removed(tmp_path):
+    async def go():
+        msgs = {i: old_msg(i, f"post {i}") for i in range(1, 8)}
+        chan, db, ch = await world(tmp_path, msgs)
+        chan.msgs.clear()  # e.g. the bot lost access and Telegram shows nothing
+        rep = await sync_channel(chan, db, ch, opts())
+        assert rep.deleted == [] and rep.anomaly and "All 7" in rep.anomaly
+        assert len(await db.sent_posts(1)) == 7
+        await db.close()
+
+    run(go())
+
+
+def test_one_or_two_posts_can_be_deleted_even_when_that_is_all_there_is(tmp_path):
+    async def go():
+        chan, db, ch = await world(tmp_path, {1: old_msg(1, "a"), 2: old_msg(2, "b")})
+        chan.msgs.clear()
+        rep = await sync_channel(chan, db, ch, opts())
+        assert sorted(rep.deleted) == [1, 2] and rep.anomaly is None
+        await db.close()
+
+    run(go())
+
+
+def test_a_change_made_through_the_bot_while_the_channel_was_read_wins(tmp_path):
+    async def go():
+        chan, db, ch = await world(tmp_path, three())
+        chan.msgs[3].message = "an admin's text"
+        pid = (await db.find_by_message(1, 3)).id
+        real_get = chan.get_messages
+
+        async def get_while_the_owner_edits(peer, ids=None):
+            res = await real_get(peer, ids=ids)
+            await db.update_post(pid, text="the owner's newer text")  # happens between reading and saving
+            return res
+
+        chan.get_messages = get_while_the_owner_edits
+        rep = await sync_channel(chan, db, ch, opts())
+        assert rep.edited == [] and rep.changed_meanwhile == 1
+        assert (await db.find_by_message(1, 3)).text == "the owner's newer text"
+        await db.close()
+
+    run(go())
+
+
+# ================================================================================================= new posts
+async def looked(tmp_path, msgs=None):
+    chan, db, ch = await world(tmp_path, msgs or three())
+    first = await sync_channel(chan, db, ch, opts())  # the first look sets the mark
+    assert first.first_look
+    return chan, db, ch
+
+
+def test_a_new_post_made_outside_the_bot_is_added(tmp_path):
+    async def go():
+        chan, db, ch = await looked(tmp_path)
+        m = foreign(chan, "posted from the Telegram app", entities=[types.MessageEntityBold(0, 6)],
+                    markup=markup([url_btn("Open", "https://example.org/x")]))
+        rep = await sync_channel(chan, db, ch, opts())
+        assert rep.adopted == [m.id] and rep.first_look is False
+        p = await db.find_by_message(1, m.id)
+        assert (p.source, p.status, p.created_by) == ("adopted", "sent", 0)
+        assert p.text == "posted from the Telegram app"
+        assert p.entities == [{"k": "bold", "o": 0, "l": 6}]
+        assert p.buttons == [[{"t": "Open", "u": "https://example.org/x"}]]
+        assert p.media_kind is None and p.link_preview is False
+        assert as_utc(p.sent_at) == as_utc(m.date)
+        state = await db.get_sync(1)
+        assert state.last_top == m.id
+        again = await sync_channel(chan, db, ch, opts())
+        assert again.adopted == []  # not saved twice
+        assert len(await db.sent_posts(1)) == 4
+        await db.close()
+
+    run(go())
+
+
+def test_new_media_posts_keep_their_media(tmp_path):
+    async def go():
+        chan, db, ch = await looked(tmp_path)
+        m = foreign(chan, "caption", media=photo_media(31))
+        await sync_channel(chan, db, ch, opts())
+        p = await db.find_by_message(1, m.id)
+        assert p.media_kind == "photo" and p.media_file_id.startswith("tl:") and p.text == "caption"
+        await db.close()
+
+    run(go())
+
+
+def test_the_bots_own_posts_that_are_saved_are_not_picked_up_again(tmp_path):
+    async def go():
+        chan, db, ch = await looked(tmp_path)
+        m = chan._new("made with the bot", None, None)  # out=True, like the bot's own post
+        m.date = utcnow() - datetime.timedelta(minutes=10)
+        mine = await db.create_post(channel_id=1, message_id=m.id, status="sent", source="bot", created_by=99,
+                                    sent_at=OLD, created_at=OLD, updated_at=OLD, **live_fields(m))
+        rep = await sync_channel(chan, db, ch, opts())
+        assert rep.adopted == [] and rep.deleted == []
+        assert (await db.find_by_message(1, m.id)).id == mine.id and mine.source == "bot"
+        await db.close()
+
+    run(go())
+
+
+def test_a_message_that_appeared_a_moment_ago_waits_for_the_next_look(tmp_path):
+    async def go():
+        chan, db, ch = await looked(tmp_path)
+        m = foreign(chan, "brand new", age=1)
+        rep = await sync_channel(chan, db, ch, opts(grace=120))
+        assert rep.adopted == [] and rep.young_new == [m.id]
+        state = await db.get_sync(1)
+        assert state.last_top == 3 and state.deferred == 1  # the mark stays below it
+        m.date = utcnow() - datetime.timedelta(minutes=10)  # time passes
+        rep = await sync_channel(chan, db, ch, opts(grace=120))
+        assert rep.adopted == [m.id] and rep.young_new == []
+        state = await db.get_sync(1)
+        assert state.last_top == m.id and state.deferred == 0
+        await db.close()
+
+    run(go())
+
+
+def test_a_quiet_channel_costs_no_reading_at_all(tmp_path):
+    async def go():
+        chan, db, ch = await looked(tmp_path)
+        before = len(chan.requests)
+        reads = []
+        real_get = chan.get_messages
+
+        async def counting(peer, ids=None):
+            reads.append(list(ids))
+            return await real_get(peer, ids=ids)
+
+        chan.get_messages = counting
+        rep = await sync_channel(chan, db, ch, opts())
+        assert rep.adopted == [] and rep.scanned_to == 3  # only the saved posts were compared (ids 1-3)
+        assert reads == [[1, 2, 3]]  # nothing was read beyond them: not a single event since the last look
+        assert len(chan.requests) - before == 1  # just the channel counter
+        await db.close()
+
+    run(go())
+
+
+def test_messages_of_a_kind_the_bot_cannot_hold_are_left_alone(tmp_path):
+    async def go():
+        chan, db, ch = await looked(tmp_path)
+        foreign(chan, "")  # a poll or a sticker: no text, no supported media
+        rep = await sync_channel(chan, db, ch, opts())
+        assert rep.adopted == [] and rep.unsupported == 1
+        await db.close()
+
+    run(go())
+
+
+def test_a_post_the_owner_made_the_bot_forget_is_not_brought_back(tmp_path):
+    async def go():
+        chan, db, ch = await looked(tmp_path)
+        m = foreign(chan, "a post that cannot be deleted")
+        rep = await sync_channel(chan, db, ch, opts(ignore={(1, m.id)}))
+        assert rep.adopted == []
+        await db.close()
+
+    run(go())
+
+
+def test_a_dead_tail_longer_than_the_quick_look_needs_a_deep_look(tmp_path):
+    async def go():
+        chan, db, ch = await world(tmp_path, three())
+        chan._top, chan._pts = 3, 900  # lots of events happened: the counter is far above the newest message
+        first = await sync_channel(chan, db, ch, opts())
+        assert (await db.get_sync(1)).last_top == 3 and first.first_look
+        chan._top = 600  # 597 ids went to messages that were deleted, then a new post appears
+        chan._pts = 901
+        m = foreign(chan, "after a very long run of deleted ids")
+        assert m.id == 601
+        quick = await sync_channel(chan, db, ch, opts())
+        assert quick.adopted == []  # the quick look reads 300 ids beyond the likely end and stops
+        deep = await sync_channel(chan, db, ch, opts(deep=True))
+        assert deep.adopted == [601]
+        assert (await db.get_sync(1)).last_top == 601
+        await db.close()
+
+    run(go())
+
+
+def test_the_automatic_looks_go_deep_once_a_day(tmp_path):
+    async def go():
+        chan, db, ch = await world(tmp_path, three())
+        chan._top, chan._pts = 3, 900
+        await sync_channel(chan, db, ch, opts())
+        chan._top, chan._pts = 600, 901
+        m = foreign(chan, "hidden behind a long dead tail")
+        assert (await sync_channel(chan, db, ch, opts())).adopted == []
+        # a day later the same automatic look reads every possible id
+        old = utcnow() - datetime.timedelta(hours=25)
+        state = await db.get_sync(1)
+        await db.save_sync(1, last_top=state.last_top, last_pts=state.last_pts, deep=False)
+        async with db.Session() as s:
+            row = await s.get(type(state), 1)
+            row.last_deep = old
+            await s.commit()
+        assert (await sync_channel(chan, db, ch, opts())).adopted == [m.id]
+        await db.close()
+
+    run(go())
+
+
+# ======================================================================================== service messages
+def test_service_messages_that_appeared_are_deleted_by_the_look(tmp_path):
+    async def go():
+        chan, db, ch = await looked(tmp_path)
+        chan.msgs[4] = service_msg(4)
+        chan.msgs[5] = types.MessageService(id=5, peer_id=types.PeerChannel(1), date=OLD, action=types.MessageActionChatEditTitle("New"))
+        chan._top, chan._pts = 5, 5
+        rep = await sync_channel(chan, db, ch, opts())
+        assert rep.services_deleted == 2 and rep.services_failed == 0
+        assert 4 not in chan.msgs and 5 not in chan.msgs and sorted(chan.msgs) == [1, 2, 3]
+        assert rep.adopted == []
+        await db.close()
+
+    run(go())
+
+
+def test_service_messages_stay_when_the_switch_is_off(tmp_path):
+    async def go():
+        chan, db, ch = await looked(tmp_path)
+        chan.msgs[4] = service_msg(4)
+        chan._top, chan._pts = 4, 4
+        rep = await sync_channel(chan, db, ch, opts(delete_services=False))
+        assert rep.services_deleted == 0 and 4 in chan.msgs
+        await db.close()
+
+    run(go())
+
+
+def test_a_service_message_the_bot_may_not_delete_is_counted(tmp_path):
+    async def go():
+        chan, db, ch = await looked(tmp_path)
+        chan.msgs[4] = service_msg(4)
+        chan._top, chan._pts = 4, 4
+        chan.undeletable.add(4)
+        rep = await sync_channel(chan, db, ch, opts())
+        assert (rep.services_deleted, rep.services_failed) == (0, 1) and rep.services_error
+        await db.close()
+
+    run(go())
+
+
+def test_clean_sweeps_the_whole_history_for_old_service_messages(tmp_path):
+    async def go():
+        msgs = {1: old_msg(1, "a"), 2: service_msg(2), 3: old_msg(3, "b"), 4: service_msg(4), 5: old_msg(5, "c")}
+        chan, db, ch = await world(tmp_path, {i: m for i, m in msgs.items() if not isinstance(m, types.MessageService)})
+        for i in (2, 4):
+            chan.msgs[i] = msgs[i]
+        rep = await sync_channel(chan, db, ch, opts(clean=True))
+        assert rep.services_deleted == 2 and sorted(chan.msgs) == [1, 3, 5]
+        assert len(await db.sent_posts(1)) == 3  # My posts untouched
+        await db.close()
+
+    run(go())
+
+
+def test_clean_hands_what_the_bot_cannot_delete_to_the_userbot(tmp_path):
+    async def go():
+        chan, db, ch = await world(tmp_path, {1: old_msg(1, "a")})
+        chan.msgs[2] = service_msg(2)
+        chan._top = chan._pts = 2
+        chan.undeletable.add(2)
+        asked = []
+
+        async def userbot_delete(ids):
+            asked.append(list(ids))
+            chan.msgs.pop(2, None)
+
+        rep = await sync_channel(chan, db, ch, opts(clean=True, fallback=userbot_delete))
+        assert asked == [[2]] and rep.services_deleted == 1 and rep.services_failed == 0
+        await db.close()
+
+    run(go())
+
+
+# ============================================================================================= errors and busy
+def test_a_channel_the_bot_cannot_read_is_reported_not_raised(tmp_path):
+    async def go():
+        chan, db, ch = await world(tmp_path, three())
+        chan.fail["GetFullChannelRequest"] = errors.ChannelPrivateError(request=None)
+        rep = await sync_channel(chan, db, ch, opts())
+        assert rep.error and "ChannelPrivateError" in rep.error
+        assert len(await db.sent_posts(1)) == 3
+        chan.fail.clear()
+        chan.fail["GetMessagesRequest"] = errors.ChannelPrivateError(request=None)
+        rep = await sync_channel(chan, db, ch, opts())
+        assert rep.error and len(await db.sent_posts(1)) == 3
+        await db.close()
+
+    run(go())
+
+
+def test_a_long_job_that_starts_cuts_the_look_short(tmp_path):
+    async def go():
+        chan, db, ch = await looked(tmp_path)
+        m = foreign(chan, "new while a repost runs")
+        del chan.msgs[2]
+        chan._pts += 1
+        rep = await sync_channel(chan, db, ch, opts(), busy=lambda: True)
+        assert rep.busy and rep.deleted == [] and rep.adopted == []
+        assert len(await db.sent_posts(1)) == 3
+        rep = await sync_channel(chan, db, ch, opts())  # the job is over
+        assert rep.deleted == [2] and rep.adopted == [m.id]
+        await db.close()
+
+    run(go())
+
+
+# =================================================================================== looking at given ids only
+def test_sync_ids_handles_edits_deletes_new_posts_and_notices(tmp_path):
+    async def go():
+        chan, db, ch = await world(tmp_path, three())
+        chan.msgs[1].message = "edited"
+        del chan.msgs[3]
+        new = foreign(chan, "added")
+        chan.msgs[9] = service_msg(9)
+        chan._top = 9
+        ids = [1, 3, new.id, 9, 2]
+        rep = await sync_ids(chan, db, ch, ids, opts(), adopt_ids={new.id, 9})
+        assert rep.edited == [1] and rep.deleted == [3] and rep.adopted == [new.id]
+        assert rep.services_deleted == 1 and 9 not in chan.msgs
+        assert (await db.find_by_message(1, 2)).text == "second"
+        await db.close()
+
+    run(go())
+
+
+def test_an_edit_of_a_post_the_bot_never_saved_is_history_and_ignored(tmp_path):
+    async def go():
+        chan, db, ch = await world(tmp_path, three())
+        old = old_msg(7, "an old post, edited today")
+        chan.msgs[7] = old
+        chan._top = 7
+        rep = await sync_ids(chan, db, ch, [7], opts(), adopt_ids=set())
+        assert rep.adopted == [] and await db.find_by_message(1, 7) is None
+        await db.close()
+
+    run(go())
+
+
+def test_sync_ids_reports_what_is_too_fresh(tmp_path):
+    async def go():
+        chan, db, ch = await world(tmp_path, three())
+        new = foreign(chan, "right now", age=0)
+        await db.update_post((await db.find_by_message(1, 1)).id, text="saved a second ago")
+        chan.msgs[1].message = "different"
+        rep = await sync_ids(chan, db, ch, [1, new.id], opts(grace=60), adopt_ids={new.id})
+        assert rep.young == [1] and rep.young_new == [new.id] and rep.adopted == [] and rep.edited == []
+        await db.close()
+
+    run(go())
+
+
+# ================================================================================================ the database
+def test_adopting_a_message_twice_saves_it_once(tmp_path):
+    async def go():
+        db = await fresh_db(db_url_for(tmp_path))
+        await db.save_channel(1, 5, "Chan", "chan", 99)
+        p, created = await db.adopt_message(1, 50, {"text": "x"}, sent_at=OLD)
+        assert created and p.source == "adopted"
+        again, created = await db.adopt_message(1, 50, {"text": "y"})
+        assert again is None and created is False
+        assert (await db.find_by_message(1, 50)).text == "x"
+        await db.close()
+
+    run(go())
+
+
+def test_a_reposted_copy_takes_the_place_of_a_post_the_sync_saved_first(tmp_path):
+    async def go():
+        db = await fresh_db(db_url_for(tmp_path))
+        await db.save_channel(1, 5, "Chan", "chan", 99)
+        await db.adopt_message(1, 50, {"text": "picked up by the sync"})
+        await db.create_migration("m1", 1, old=None, new=None, include_typed=False, include_posts=False, first_id=1, last_id=9, partial=False, user_id=99)
+        await db.record_repost("m1", 1, [{"old_id": 1, "new_id": 50, "post": {"text": "the copy"}}], 99)
+        p = await db.find_by_message(1, 50)
+        assert p.text == "the copy" and p.source == "bot"
+        await db.close()
+
+    run(go())
+
+
+def test_publishing_takes_the_place_of_a_post_the_sync_saved_first(tmp_path):
+    async def go():
+        db = await fresh_db(db_url_for(tmp_path))
+        await db.save_channel(1, 5, "Chan", "chan", 99)
+        await db.adopt_message(1, 60, {"text": "picked up first"})
+        draft = await db.create_post(channel_id=1, status="draft", text="my draft", created_by=99)
+        assert await db.release_slot(1, 60, keep_pid=draft.id) == 1
+        sent = await db.update_post(draft.id, status="sent", message_id=60)
+        assert sent.message_id == 60 and (await db.find_by_message(1, 60)).text == "my draft"
+        await db.close()
+
+    run(go())
+
+
+def test_the_sync_update_only_applies_to_an_unchanged_post(tmp_path):
+    async def go():
+        db = await fresh_db(db_url_for(tmp_path))
+        await db.save_channel(1, 5, "Chan", "chan", 99)
+        post = await db.create_post(channel_id=1, message_id=5, status="sent", text="v1", created_by=99)
+        seen = (await db.get_post(post.id)).updated_at
+        assert await db.sync_update_post(post.id, seen, {"text": "v2"}) is True
+        assert await db.sync_update_post(post.id, seen, {"text": "v3"}) is False  # it changed since `seen`
+        assert (await db.get_post(post.id)).text == "v2"
+        await db.close()
+
+    run(go())
+
+
+def test_servicemsg_remove_reports_refusals_instead_of_raising(tmp_path):
+    async def go():
+        chan = FakeChannelClient({1: service_msg(1), 2: service_msg(2)})
+        peer = types.InputPeerChannel(1, 5)
+        ok = await remove(chan, peer, [1, 2])
+        assert (ok.deleted, ok.failed, ok.error) == (2, 0, None)
+        chan.msgs[3] = service_msg(3)
+        chan.fail["DeleteMessagesRequest"] = errors.ChatAdminRequiredError(request=None)
+        bad = await remove(chan, peer, [3])
+        assert bad.failed == 1 and bad.error_name == "ChatAdminRequiredError"
+
+    run(go())

tests/test_syncer.py

@@ -0,0 +1,643 @@
+"""The live side of the sync (what Telegram's updates say), the timed check, service-message clean-up and /sync."""
+import asyncio
+import datetime
+from types import SimpleNamespace
+
+from telethon import errors, types
+
+from app.db import utcnow
+from app.syncer import Syncer
+
+from .fakes import FakeChannelClient, markup, photo_media, url_btn
+from .harness import App, norm
+from .test_sync import OLD, foreign, old_msg, saved
+
+NOW = datetime.datetime.now(datetime.timezone.utc)
+
+
+def run(coro):
+    return asyncio.run(coro)
+
+
+# ------------------------------------------------------------------------------------------------ set-up
+async def make_app(tmp_path, *, posts=(), **cfg):
+    """Channel 1 "Anime Channel" (the app's own fake) with `posts` saved the way the bot saved them a day ago."""
+    app = App(tmp_path, edit_delay=0, **cfg)
+    await app.start()
+    app.tg.username, app.tg.title = "animech", "Anime Channel"
+    await app.db.save_channel(1, 5, "Anime Channel", "animech", 1)
+    for m in posts:
+        app.tg.msgs[m.id] = m
+        app.tg._top = app.tg._pts = max(app.tg._top, m.id)
+        await saved(app.db, m)
+    sy = app.ctx.syncer
+    sy.debounce, sy.max_debounce, sy.lock_wait, sy.rt_grace, sy.pause, sy.start_delay = 0.01, 0.2, 0.02, 0, 0, 0.01
+    return app
+
+
+def three():
+    return [
+        old_msg(1, "first"),
+        old_msg(2, "second", markup=markup([url_btn("Watch", "https://t.me/a/1")])),
+        old_msg(3, "third"),
+    ]
+
+
+def new_update(m):
+    return types.UpdateNewChannelMessage(message=m, pts=m.id, pts_count=1)
+
+
+def edit_update(m):
+    return types.UpdateEditChannelMessage(message=m, pts=m.id, pts_count=1)
+
+
+def delete_update(cid, ids):
+    return types.UpdateDeleteChannelMessages(channel_id=cid, messages=list(ids), pts=0, pts_count=len(ids))
+
+
+def notice(i, action=None, cid=1):
+    return types.MessageService(
+        id=i, peer_id=types.PeerChannel(cid), date=NOW, action=action or types.MessageActionPinMessage()
+    )
+
+
+def dms(app):
+    """Texts the bot sent to people (not to channels)."""
+    return [e[2] for e in app.tg.sent if not isinstance(e[1], types.InputPeerChannel)]
+
+
+# ============================================================================================ service messages
+def test_every_kind_of_notice_is_deleted_the_moment_it_appears(tmp_path):
+    async def go():
+        app = await make_app(tmp_path, posts=three())
+        actions = [
+            types.MessageActionChatEditTitle("A new name"),
+            types.MessageActionChatEditPhoto(photo=types.PhotoEmpty(0)),
+            types.MessageActionChatDeletePhoto(),
+            types.MessageActionPinMessage(),
+            types.MessageActionGroupCall(call=types.InputGroupCall(1, 2)),
+            types.MessageActionHistoryClear(),
+        ]
+        for n, action in enumerate(actions, start=10):
+            app.tg.msgs[n] = notice(n, action)
+            await app.raw(new_update(app.tg.msgs[n]))
+        assert [i for i in app.tg.msgs if i >= 10] == []
+        assert sorted(app.tg.msgs) == [1, 2, 3]  # posts are never touched
+        assert len(await app.db.sent_posts(1)) == 3
+        await app.db.close()
+
+    run(go())
+
+
+def test_notices_stay_when_the_switch_is_off(tmp_path):
+    async def go():
+        app = await make_app(tmp_path, posts=three(), delete_service_messages=False)
+        app.tg.msgs[10] = notice(10)
+        await app.raw(new_update(app.tg.msgs[10]))
+        assert 10 in app.tg.msgs
+        await app.db.close()
+
+    run(go())
+
+
+def test_notices_in_channels_that_are_not_connected_are_left_alone(tmp_path):
+    async def go():
+        app = await make_app(tmp_path, posts=three())
+        other = app.tg.add_channel(FakeChannelClient({5: notice(5, cid=2)}, username="other", cid=2, title="Other"))
+        before = len(app.tg.requests) + len(other.requests)
+        await app.raw(new_update(other.msgs[5]))
+        assert 5 in other.msgs and len(app.tg.requests) + len(other.requests) == before  # not even a request
+        # a channel that was removed from the bot counts as not connected too
+        await app.db.set_channel_active(1, False)
+        app.ctx.syncer.forget_cache()
+        app.tg.msgs[10] = notice(10)
+        await app.raw(new_update(app.tg.msgs[10]))
+        assert 10 in app.tg.msgs
+        await app.db.close()
+
+    run(go())
+
+
+def test_the_owner_is_told_once_when_the_bot_may_not_delete(tmp_path):
+    async def go():
+        app = await make_app(tmp_path, posts=three())
+        app.tg.fail["DeleteMessagesRequest"] = errors.ChatAdminRequiredError(request=None)
+        for i in (10, 11, 12):
+            app.tg.msgs[i] = notice(i)
+            await app.raw(new_update(app.tg.msgs[i]))
+        texts = [norm(t) for t in dms(app)]
+        assert len(texts) == 1 and "Anime Channel" in texts[0] and "Delete messages" in texts[0]
+        assert "DELETE_SERVICE_MESSAGES" in texts[0]
+        assert {10, 11, 12} <= set(app.tg.msgs)
+        # a day later the same problem is worth saying again
+        app.ctx.syncer.tell_every = 0
+        await app.raw(new_update(app.tg.msgs[10]))
+        assert len(dms(app)) == 2
+        await app.db.close()
+
+    run(go())
+
+
+def test_other_refusals_are_only_logged(tmp_path):
+    async def go():
+        app = await make_app(tmp_path, posts=three())
+        app.tg.fail["DeleteMessagesRequest"] = errors.MessageDeleteForbiddenError(request=None)
+        app.tg.msgs[10] = notice(10)
+        await app.raw(new_update(app.tg.msgs[10]))
+        assert dms(app) == [] and 10 in app.tg.msgs
+        await app.db.close()
+
+    run(go())
+
+
+def test_notices_are_deleted_even_when_the_automatic_sync_is_switched_off(tmp_path):
+    async def go():
+        app = await make_app(tmp_path, posts=three(), sync_interval_minutes=0)
+        app.tg.msgs[10] = notice(10)
+        await app.raw(new_update(app.tg.msgs[10]))
+        assert 10 not in app.tg.msgs
+        await app.db.close()
+
+    run(go())
+
+
+def test_notices_are_deleted_while_a_long_job_runs(tmp_path):
+    async def go():
+        app = await make_app(tmp_path, posts=three())
+        async with app.ctx.lock:
+            app.tg.msgs[10] = notice(10)
+            await app.raw(new_update(app.tg.msgs[10]))
+        assert 10 not in app.tg.msgs
+        await app.db.close()
+
+    run(go())
+
+
+# ============================================================================================== live changes
+def test_a_new_post_by_someone_else_is_added_a_moment_later(tmp_path):
+    async def go():
+        app = await make_app(tmp_path, posts=three())
+        sy = app.ctx.syncer
+        m = foreign(app.tg, "from the Telegram app", markup=markup([url_btn("Open", "https://example.org")]))
+        await app.raw(new_update(m))
+        assert await app.db.find_by_message(1, m.id) is None  # not yet: the wait lets the bot finish its own business
+        await sy.flush_events()
+        p = await app.db.find_by_message(1, m.id)
+        assert p.source == "adopted" and p.text == "from the Telegram app" and p.buttons == [[{"t": "Open", "u": "https://example.org"}]]
+        await app.db.close()
+
+    run(go())
+
+
+def test_the_bots_own_posts_are_not_looked_at(tmp_path):
+    async def go():
+        app = await make_app(tmp_path, posts=three())
+        sy = app.ctx.syncer
+        mine = app.tg._new("sent by the bot", None, None)  # out=True
+        await app.raw(new_update(mine))
+        assert not sy._pending()
+        await app.db.close()
+
+    run(go())
+
+
+def test_nothing_is_queued_while_a_long_job_runs_or_when_the_sync_is_off(tmp_path):
+    async def go():
+        app = await make_app(tmp_path, posts=three())
+        sy = app.ctx.syncer
+        m = foreign(app.tg, "x")
+        async with app.ctx.lock:
+            await app.raw(new_update(m))
+            await app.raw(edit_update(app.tg.msgs[1]))
+        assert not sy._pending()
+        off = await make_app(tmp_path / "off", posts=three(), sync_interval_minutes=0)
+        m2 = foreign(off.tg, "y")
+        await off.raw(new_update(m2))
+        await off.raw(edit_update(off.tg.msgs[1]))
+        await off.raw(delete_update(1, [2]))
+        assert not off.ctx.syncer._pending() and len(await off.db.sent_posts(1)) == 3
+        await app.db.close()
+        await off.db.close()
+
+    (tmp_path / "off").mkdir()
+    run(go())
+
+
+def test_an_edit_made_by_an_admin_is_recorded(tmp_path):
+    async def go():
+        app = await make_app(tmp_path, posts=three())
+        sy = app.ctx.syncer
+        m = app.tg.msgs[3]
+        m.message, m.entities = "third, with a correction", [types.MessageEntityBold(0, 5)]
+        await app.raw(edit_update(m))
+        await sy.flush_events()
+        p = await app.db.find_by_message(1, 3)
+        assert p.text == "third, with a correction" and p.entities == [{"k": "bold", "o": 0, "l": 5}]
+        assert sy.last_reports[1].edited == [3]
+        await app.db.close()
+
+    run(go())
+
+
+def test_the_echo_of_an_edit_made_through_the_bot_changes_nothing(tmp_path):
+    async def go():
+        app = await make_app(tmp_path, posts=three())
+        sy = app.ctx.syncer
+        sy.rt_grace = 60  # the bot saved the post a moment ago
+        pid = (await app.db.find_by_message(1, 1)).id
+        await app.db.update_post(pid, text="the owner's new text")  # updated_at = now
+        app.tg.msgs[1].message = "the owner's new text"
+        await app.raw(edit_update(app.tg.msgs[1]))
+        sy.max_tries = 1
+        await sy.flush_events()
+        assert (await app.db.find_by_message(1, 1)).text == "the owner's new text"
+        assert sy.last_reports[1].edited == []
+        await app.db.close()
+
+    run(go())
+
+
+def test_a_post_deleted_by_an_admin_leaves_my_posts_at_once(tmp_path):
+    async def go():
+        app = await make_app(tmp_path, posts=three())
+        sy = app.ctx.syncer
+        m = foreign(app.tg, "will be deleted before the wait is over")
+        await app.raw(new_update(m))
+        await app.raw(edit_update(app.tg.msgs[2]))
+        del app.tg.msgs[2]
+        await app.raw(delete_update(1, [2, m.id]))
+        assert await app.db.find_by_message(1, 2) is None  # no flush needed
+        assert sy._news == {1: set()} and sy._edits == {1: set()}  # and nothing is left to look at for them
+        await sy.flush_events()
+        assert await app.db.find_by_message(1, m.id) is None
+        assert sorted(p.message_id for p in await app.db.sent_posts(1)) == [1, 3]
+        await app.db.close()
+
+    run(go())
+
+
+def test_deleted_posts_of_other_channels_are_ignored(tmp_path):
+    async def go():
+        app = await make_app(tmp_path, posts=three())
+        await app.raw(delete_update(77, [1, 2, 3]))
+        assert len(await app.db.sent_posts(1)) == 3
+        await app.db.close()
+
+    run(go())
+
+
+def test_a_message_that_is_still_too_fresh_is_looked_at_again(tmp_path):
+    async def go():
+        app = await make_app(tmp_path, posts=three())
+        sy = app.ctx.syncer
+        sy.rt_grace, sy.max_tries = 0.15, 500
+        m = foreign(app.tg, "just posted", age=0)
+        await app.raw(new_update(m))
+        await asyncio.wait_for(sy.flush_events(), 5)
+        p = await app.db.find_by_message(1, m.id)
+        assert p is not None and p.source == "adopted"
+        assert not sy._pending() and sy._tries == {}
+        await app.db.close()
+
+    run(go())
+
+
+def test_a_message_that_never_gets_old_enough_is_given_up(tmp_path):
+    async def go():
+        app = await make_app(tmp_path, posts=three())
+        sy = app.ctx.syncer
+        sy.rt_grace, sy.max_tries = 3600, 2
+        m = foreign(app.tg, "from the future", age=0)
+        await app.raw(new_update(m))
+        await asyncio.wait_for(sy.flush_events(), 5)
+        assert await app.db.find_by_message(1, m.id) is None and not sy._pending() and sy._tries == {}
+        await app.db.close()
+
+    run(go())
+
+
+def test_the_queue_waits_for_a_long_job_to_finish(tmp_path):
+    async def go():
+        app = await make_app(tmp_path, posts=three())
+        sy = app.ctx.syncer
+        m = foreign(app.tg, "posted before the long job started")
+        await app.raw(new_update(m))
+        async with app.ctx.lock:
+            task = asyncio.create_task(sy.flush_events())
+            await asyncio.sleep(0.15)
+            assert not task.done() and await app.db.find_by_message(1, m.id) is None
+        await asyncio.wait_for(task, 5)
+        assert (await app.db.find_by_message(1, m.id)).source == "adopted"
+        await app.db.close()
+
+    run(go())
+
+
+def test_a_burst_of_changes_becomes_one_look_at_the_whole_channel(tmp_path):
+    async def go():
+        app = await make_app(tmp_path, posts=three())
+        sy = app.ctx.syncer
+        sy.max_ids = 2
+        for i in (1, 2, 3):
+            app.tg.msgs[i].message = f"edited {i}"
+            await app.raw(edit_update(app.tg.msgs[i]))
+        await sy.flush_events()
+        rep = sy.last_reports[1]
+        assert rep.checked == 3 and sorted(rep.edited) == [1, 2, 3]  # compared as a whole, not id by id
+        await app.db.close()
+
+    run(go())
+
+
+# ===================================================================================================== the timer
+def test_the_timed_check_looks_at_every_channel_and_gives_way_to_long_jobs(tmp_path):
+    async def go():
+        app = await make_app(tmp_path, posts=three())
+        sy = app.ctx.syncer
+        app.tg.msgs[2].message = "edited meanwhile"
+        async with app.ctx.lock:
+            assert await sy.run_all() == []  # a long job is running: nothing is read
+        assert (await app.db.find_by_message(1, 2)).text == "second"
+        reports = await sy.run_all()
+        assert [r.channel.id for r in reports] == [1] and reports[0].edited == [2]
+        assert sy.last_reports[1] is reports[0]
+        await app.db.close()
+
+    run(go())
+
+
+def test_one_channel_that_fails_does_not_stop_the_others(tmp_path):
+    async def go():
+        app = await make_app(tmp_path, posts=three())
+        broken = app.tg.add_channel(FakeChannelClient({1: old_msg(1, "x")}, username="broken", cid=2, title="Broken"))
+        await app.db.save_channel(2, 6, "Broken", "broken", 1)
+        await app.db.create_post(channel_id=2, message_id=1, status="sent", source="bot", created_by=1, text="x",
+                                 created_at=OLD, updated_at=OLD, sent_at=OLD)
+        broken.fail["GetMessagesRequest"] = RuntimeError("the network dropped")
+        app.tg.msgs[3].message = "edited"
+        reports = await app.ctx.syncer.run_all()
+        by_id = {r.channel.id: r for r in reports}
+        assert by_id[1].edited == [3] and by_id[1].error is None
+        assert by_id[2].error and "RuntimeError" in by_id[2].error
+        await app.db.close()
+
+    run(go())
+
+
+def test_the_timer_starts_and_stops_with_the_bot(tmp_path):
+    async def go():
+        off = await make_app(tmp_path / "off", posts=three(), sync_interval_minutes=0)
+        off.ctx.syncer.start()
+        assert off.ctx.syncer._tasks == [] and off.ctx.syncer.enabled is False
+
+        app = await make_app(tmp_path, posts=three())
+        sy = app.ctx.syncer
+        app.tg.msgs[1].message = "edited"
+        sy.start()
+        assert len(sy._tasks) == 2
+        for _ in range(100):
+            if 1 in sy.last_reports:
+                break
+            await asyncio.sleep(0.05)
+        assert sy.last_reports[1].edited == [1]
+        await sy.stop()
+        assert sy._tasks == []
+        await app.db.close()
+        await off.db.close()
+
+    (tmp_path / "off").mkdir()
+    run(go())
+
+
+def test_the_syncer_is_part_of_the_context_the_bot_builds(tmp_path):
+    async def go():
+        app = await make_app(tmp_path)
+        assert isinstance(app.ctx.syncer, Syncer)
+        await app.db.close()
+
+    run(go())
+
+
+# ============================================================================================ the /sync command
+def test_sync_is_for_the_owner_only(tmp_path):
+    async def go():
+        app = await make_app(tmp_path, posts=three())
+        app.uid = 555
+        await app.text("/sync")
+        assert "private" in norm(app.out.last_text).lower() or "owner" in norm(app.out.last_text).lower()
+        assert app.ctx.syncer.last_reports == {}
+        await app.db.close()
+
+    run(go())
+
+
+def test_sync_needs_a_channel_and_understands_only_its_options(tmp_path):
+    async def go():
+        bare = App(tmp_path, edit_delay=0)
+        await bare.start()
+        await bare.text("/sync")
+        assert "No channels registered" in norm(bare.out.last_text)
+        await bare.db.close()
+
+        app = await make_app(tmp_path / "b", posts=three())
+        await app.text("/sync --bogus")
+        assert "I don't know “--bogus”" in norm(app.out.last_text) and "Usage: /sync" in norm(app.out.last_text)
+        await app.text("/sync --channel @nowhere")
+        assert "isn't registered" in norm(app.out.last_text)
+        await app.db.close()
+
+    (tmp_path / "b").mkdir()
+    run(go())
+
+
+def test_sync_reports_what_it_found_in_every_channel(tmp_path):
+    async def go():
+        app = await make_app(tmp_path, posts=three())
+        await app.text("/sync")  # the first look
+        first = norm(app.out.last_text)
+        assert "Sync finished" in first and "Anime Channel" in first and "First look" in first
+        assert "3 saved post(s) compared" in first
+
+        app.tg.msgs[2].message = "second, edited"
+        del app.tg.msgs[3]
+        app.tg._pts += 2
+        new = foreign(app.tg, "from the app")
+        app.tg.msgs[new.id + 1] = notice(new.id + 1)
+        app.tg._top = new.id + 1
+        app.tg._pts += 1
+        app.tg.msgs[2].reply_markup = None  # and the channel shows no keyboard for post 2
+        await app.text("/sync")
+        text = norm(app.out.last_text)
+        assert "1 deleted in the channel" in text and "message 3" in text
+        assert "1 edited in the channel" in text and "message 2" in text
+        assert "1 new post(s) made outside the bot" in text and f"message {new.id}" in text
+        assert "1 service message(s) deleted" in text
+        assert "show no buttons in the channel (message 2)" in text and "Check buttons" in text
+        assert not app.ctx.lock.locked()
+        await app.text("/sync")
+        assert "Everything matches." in norm(app.out.last_text) or "show no buttons" in norm(app.out.last_text)
+        await app.db.close()
+
+    run(go())
+
+
+def test_sync_looks_at_one_channel_only_when_asked(tmp_path):
+    async def go():
+        app = await make_app(tmp_path, posts=three())
+        app.tg.add_channel(FakeChannelClient({}, username="movies", cid=2, title="Movies"))
+        await app.db.save_channel(2, 6, "Movies", "movies", 1)
+        await app.text("/sync --channel @movies")
+        text = norm(app.out.last_text)
+        assert "Movies" in text and "Anime Channel" not in text
+        assert set(app.ctx.syncer.last_reports) == {2}
+        await app.db.close()
+
+    run(go())
+
+
+def test_sync_holds_the_long_job_lock_and_refuses_a_second_run(tmp_path):
+    async def go():
+        app = await make_app(tmp_path, posts=three())
+        gate = asyncio.Event()
+        real_get = app.tg.get_messages
+        seen = []
+
+        async def slow(peer, ids=None):
+            seen.append(app.ctx.lock.locked())
+            await gate.wait()
+            return await real_get(peer, ids=ids)
+
+        app.tg.get_messages = slow
+        first = asyncio.create_task(app.text("/sync"))
+        await asyncio.sleep(0.1)
+        await app.text("/sync")
+        assert "still running" in norm(app.out.last_text)
+        gate.set()
+        await asyncio.wait_for(first, 5)
+        assert seen and all(seen) and not app.ctx.lock.locked()
+        await app.db.close()
+
+    run(go())
+
+
+def test_sync_clean_removes_the_old_notices_from_the_history(tmp_path):
+    async def go():
+        app = await make_app(tmp_path, posts=three())
+        for i in (4, 5, 6):
+            app.tg.msgs[i] = notice(i)
+        app.tg._top = app.tg._pts = 6
+        await app.text("/sync")  # a plain look does not go through the history
+        assert {4, 5, 6} <= set(app.tg.msgs)
+        await app.text("/sync --clean")
+        text = norm(app.out.last_text)
+        assert "service-message clean-up" in text and "3 service message(s) deleted" in text
+        assert sorted(app.tg.msgs) == [1, 2, 3]
+        await app.db.close()
+
+    run(go())
+
+
+def test_sync_clean_asks_the_userbot_for_what_the_bot_may_not_delete(tmp_path):
+    async def go():
+        app = await make_app(tmp_path, posts=three())
+        app.tg.msgs[4] = notice(4)
+        app.tg._top = app.tg._pts = 4
+        app.tg.undeletable.add(4)
+        log = SimpleNamespace(asked=[], closed=0)
+
+        async def delete(ids):
+            log.asked.append(list(ids))
+            app.tg.msgs.pop(4, None)
+
+        async def close():
+            log.closed += 1
+
+        async def fallback_for(ch):
+            return delete, SimpleNamespace(close=close)
+
+        app.ctx.userbot = SimpleNamespace(fallback_for=fallback_for)
+        await app.text("/sync")
+        assert log.asked == [] and 4 in app.tg.msgs  # a plain look never calls the userbot
+        await app.text("/sync --clean")
+        assert log.asked == [[4]] and log.closed == 1 and "1 service message(s) deleted" in norm(app.out.last_text)
+        await app.db.close()
+
+    run(go())
+
+
+def test_sync_reports_a_channel_it_cannot_read(tmp_path):
+    async def go():
+        app = await make_app(tmp_path, posts=three())
+        app.tg.fail["GetFullChannelRequest"] = errors.ChannelPrivateError(request=None)
+        await app.text("/sync")
+        text = norm(app.out.last_text)
+        assert "Anime Channel" in text and "ChannelPrivateError" in text
+        assert len(await app.db.sent_posts(1)) == 3
+        await app.db.close()
+
+    run(go())
+
+
+# ================================================================================== the rest of the bot
+def test_forgetting_a_post_in_the_bot_keeps_the_sync_from_bringing_it_back(tmp_path):
+    async def go():
+        app = await make_app(tmp_path, posts=three())
+        p = await app.db.find_by_message(1, 2)
+        await app.press(f"dly:{p.id}:b")
+        assert await app.db.find_by_message(1, 2) is None and (1, 2) in app.ctx.sync_ignore
+        assert 2 in app.tg.msgs  # still in the channel
+        await app.text("/sync")  # the first look
+        del app.tg.msgs[2]
+        assert (1, 2) in app.ctx.sync_ignore
+        await app.db.close()
+
+    run(go())
+
+
+def test_a_post_the_sync_saved_first_does_not_stop_the_bot_from_publishing(tmp_path):
+    async def go():
+        app = await make_app(tmp_path, posts=three())
+        draft = await app.db.create_post(channel_id=1, status="draft", text="my new post", created_by=1)
+        sent = app.tg._new("my new post", None, None)  # what Telegram will answer the publish with
+        await app.db.adopt_message(1, sent.id, {"text": "picked up by the sync a moment earlier"})
+
+        async def send_message(peer, message="", **kw):
+            return sent
+
+        app.tg.send_message = send_message
+        await app.press(f"pby:{draft.id}")
+        p = await app.db.find_by_message(1, sent.id)
+        assert p.id == draft.id and p.source == "bot" and p.text == "my new post"
+        await app.db.close()
+
+    run(go())
+
+
+def test_the_panel_says_where_an_adopted_post_came_from(tmp_path):
+    from app.ui import panel_text
+
+    async def go():
+        app = await make_app(tmp_path, posts=three())
+        p, _ = await app.db.adopt_message(1, 40, {"text": "made in the app"})
+        ch = await app.db.get_channel(1)
+        assert "not made with this bot" in panel_text(p, ch)
+        await app.db.close()
+
+    run(go())
+
+
+def test_photos_of_new_posts_survive_the_trip_to_my_posts(tmp_path):
+    async def go():
+        app = await make_app(tmp_path, posts=three())
+        m = foreign(app.tg, "a captioned photo", media=photo_media(41))
+        await app.raw(new_update(m))
+        await app.ctx.syncer.flush_events()
+        p = await app.db.find_by_message(1, m.id)
+        assert p.media_kind == "photo" and p.media_file_id.startswith("tl:")
+        await app.db.close()
+
+    run(go())
+
+
+def test_utcnow_is_what_the_fresh_checks_compare_with():
+    assert (utcnow() - NOW).total_seconds() < 3600  # sanity for the helpers above

tests/test_units.py

@@ -9,7 +9,7 @@ from telethon.sessions import MemorySession
 from app import buttons as B
 from app.common import UserError
 from app.db import normalize_db_url
-from app.handlers.channels import parse_channel_ref
+from app.channelref import parse_channel_ref
 from app.handlers.panel import extract_content
 from app.handlers.replace import parse_replace_args
 from app.linkswap import apply_span_replacements, classify_url, normalize_username, swap_url

tests/test_wire3.py

@@ -0,0 +1,293 @@
+"""Replies, pins, the copy between two channels and the sync with the channels, through a REAL Telethon client object.
+
+Telethon resolves and serialises every request exactly as it does before sending it to Telegram; only the network at
+the very end is replaced by the simulated channels. A request that would not build or serialise in production fails here.
+"""
+import asyncio
+
+import pytest
+from telethon import TelegramClient, functions, types, utils
+from telethon.sessions import MemorySession
+
+from app.db import utcnow
+from app.repost_engine import run_repost
+from app.servicemsg import remove
+from app.shift_engine import delete_shift_copies, run_shift
+from app.sync_engine import SyncOptions, sync_channel, sync_ids
+from app.tgutil import peer_of
+
+from .dbutil import db_url_for, fresh_db
+from .fakes import FakeChannelClient, markup, rpc, service_msg, url_btn
+from .test_repost_links import CH, build_linked, new_posts, reply_of
+from .test_shift import SRC, old_posts, source_posts
+from .test_sync import foreign, old_msg, saved
+
+
+class Wire:
+    """A real client whose requests are answered by `root` (a simulated channel; the other channels sit next to it)."""
+
+    def __init__(self):
+        self.client = TelegramClient(MemorySession(), 1, "h")
+        self.root = None
+        self.seen = []
+
+
+@pytest.fixture()
+def wire(monkeypatch):
+    w = Wire()
+
+    async def fake_call(self, request, ordered=False, flood_sleep_threshold=None):
+        assert not isinstance(request, (list, tuple))
+        await request.resolve(self, utils)  # what Telethon does just before sending
+        w.seen.append(request)
+        return await w.root(request)  # serialises again and answers like Telegram
+
+    monkeypatch.setattr(TelegramClient, "__call__", fake_call)
+    return w
+
+
+def two_wired(w, *, restrict=False):
+    """channel 1 "srcch" with the source posts, channel 2 "dstch" with ten old posts."""
+    src = FakeChannelClient(source_posts(), username="srcch", title="Anime Source")
+    src.restrict_forwards = restrict
+    dst = src.add_channel(FakeChannelClient(old_posts(), username="dstch", cid=2, title="Anime Backup"))
+    w.root = src
+    return src, dst
+
+
+def names(seen):
+    return [type(r).__name__ for r in seen]
+
+
+def test_replies_and_pins_of_a_repost_go_through_a_real_client(wire, tmp_path):
+    client, seen = wire.client, wire.seen
+    chan = wire.root = FakeChannelClient(build_linked(), username=CH)
+
+    async def go():
+        db = await fresh_db(db_url_for(tmp_path))
+        await db.save_channel(1, 5, "Test", CH, 99)
+        ch = await db.get_channel(1)
+        await db.create_migration("m1", 1, old=None, new=None, include_typed=False, include_posts=False, first_id=1, last_id=10, partial=False, user_id=99)
+        mig = await db.get_migration("m1")
+        res = await run_repost(client, db, ch, mig, 99, delay=0, settle_pause=0)
+        assert (res.copied_units, res.failed, res.aborted) == (8, [], None)
+        new = new_posts(chan)
+        assert (reply_of(new[14]), reply_of(new[15]), reply_of(new[16]), reply_of(new[17])) == (13, 14, 15, 15)
+        assert (res.final.pin_wanted, res.final.pinned, res.final.pin_error) == (2, 2, None)
+        assert chan.pinned_order == [15, 19] and new[15].pinned and new[19].pinned
+        assert not [m for m in chan.msgs.values() if isinstance(m, types.MessageService) and m.id > 10]  # pin notices deleted
+        await db.close()
+
+    asyncio.run(go())
+    got = names(seen)
+    for name in ("SendMessageRequest", "SendMultiMediaRequest", "UpdatePinnedMessageRequest", "DeleteMessagesRequest", "EditMessageRequest"):
+        assert name in got, (name, got)
+    # an answer is made with the new-style reply header, never with the old reply_to_msg_id field
+    replies = [r.reply_to for r in seen if isinstance(r, (functions.messages.SendMessageRequest, functions.messages.SendMultiMediaRequest)) and r.reply_to]
+    assert replies and all(isinstance(r, types.InputReplyToMessage) for r in replies)
+    pins = [r for r in seen if isinstance(r, functions.messages.UpdatePinnedMessageRequest)]
+    assert [(p.id, p.silent, bool(p.unpin)) for p in pins] == [(15, True, False), (19, True, False)]
+
+
+def test_a_shift_between_two_channels_through_a_real_client(wire, tmp_path):
+    client, seen = wire.client, wire.seen
+    src, dst = two_wired(wire)
+
+    async def go():
+        db = await fresh_db(db_url_for(tmp_path))
+        await db.save_channel(2, 6, "Anime Backup", "dstch", 1)
+        dst_row = await db.get_channel(2)
+        await db.create_shift("s1", src=SRC, dst_channel_id=2, first_id=1, last_id=8, user_id=1)
+        shift = await db.get_shift("s1")
+        res = await run_shift(client, db, shift, dst_row, 99, delay=0, settle_pause=0)
+        assert (res.copied_units, res.copied_messages, res.failed, res.aborted) == (6, 7, [], None)
+        new = {i: m for i, m in dst.msgs.items() if i > 10}
+        assert sorted(new) == [11, 12, 13, 14, 15, 16, 17]
+        assert reply_of(new[14]) == 12 and not new[14].pinned
+        assert new[17].message == "See https://t.me/dstch/11 and https://t.me/dstch/12 and https://t.me/elsewhere/9"
+        assert sorted(src.msgs) == [1, 2, 3, 4, 5, 6, 7, 8] and src.msgs[5].pinned  # the source is only read
+        assert await db.shift_pairs("s1") == [(1, 11), (2, 12), (3, 13), (5, 14), (6, 15), (7, 16), (8, 17)]
+
+        undo = await delete_shift_copies(client, db, dst_row, shift, delay=0)
+        assert (undo.deleted, undo.remaining, undo.error) == (7, 0, None)
+        assert sorted(dst.msgs) == list(range(1, 11))
+        assert sorted(src.msgs) == [1, 2, 3, 4, 5, 6, 7, 8]
+        await db.close()
+
+    asyncio.run(go())
+    # posts are carried over by Telegram itself when it is allowed to (here: from channel 1 to channel 2)
+    forwards = [r for r in seen if isinstance(r, functions.messages.ForwardMessagesRequest)]
+    assert forwards and all(isinstance(r.from_peer, types.InputPeerChannel) and r.from_peer.channel_id == 1 for r in forwards)
+    assert all(isinstance(r.to_peer, types.InputPeerChannel) and r.to_peer.channel_id == 2 for r in forwards)
+    assert all(r.drop_author for r in forwards)
+    sent = [r for r in seen if isinstance(r, functions.messages.SendMessageRequest) and r.reply_to]
+    assert sent and sent[0].peer.channel_id == 2 and sent[0].reply_to.reply_to_msg_id == 12
+    # the source channel was never written to
+    for r in seen:
+        if isinstance(r, (functions.messages.SendMessageRequest, functions.messages.SendMediaRequest, functions.messages.SendMultiMediaRequest, functions.messages.EditMessageRequest)):
+            assert r.peer.channel_id == 2
+        if isinstance(r, functions.channels.DeleteMessagesRequest):
+            assert r.channel.channel_id == 2
+
+
+def test_a_shift_from_a_channel_that_forbids_forwarding_through_a_real_client(wire, tmp_path):
+    client, seen = wire.client, wire.seen
+    src, dst = two_wired(wire, restrict=True)
+
+    async def go():
+        db = await fresh_db(db_url_for(tmp_path))
+        await db.save_channel(2, 6, "Anime Backup", "dstch", 1)
+        dst_row = await db.get_channel(2)
+        await db.create_shift("s1", src=SRC, dst_channel_id=2, first_id=1, last_id=8, user_id=1)
+        shift = await db.get_shift("s1")
+        res = await run_shift(client, db, shift, dst_row, 99, delay=0, settle_pause=0)
+        assert (res.copied_units, res.failed, res.forwarded, res.rebuilt) == (6, [], 0, 6)
+        new = {i: m for i, m in dst.msgs.items() if i > 10}
+        assert new[12].reply_markup is not None and new[15].grouped_id == new[16].grouped_id
+        await db.close()
+
+    asyncio.run(go())
+    assert "SendMultiMediaRequest" in names(seen) and "SendMediaRequest" in names(seen)
+
+
+# ===================================================================================== the sync with the channels
+def notice(chan, action):
+    """A service message as Telegram adds one: it takes the next message id and moves the event counter."""
+    chan._top += 1
+    chan._pts += 1
+    note = types.MessageService(id=chan._top, peer_id=types.PeerChannel(chan.cid), date=utcnow(), action=action)
+    chan.msgs[note.id] = note
+    return note
+
+
+def synced_channel(wire):
+    """Posts 1, 2, 3 and 5 made through the bot a day ago; 4 is an old "pinned a message" notice."""
+    msgs = {
+        1: old_msg(1, "first"),
+        2: old_msg(2, "second", markup=markup([url_btn("Watch", "https://t.me/a/1")])),
+        3: old_msg(3, "third"),
+        4: service_msg(4),
+        5: old_msg(5, "fifth"),
+    }
+    chan = wire.root = FakeChannelClient(msgs, username="chan")
+    return msgs, chan
+
+
+async def saved_world(tmp_path, msgs):
+    db = await fresh_db(db_url_for(tmp_path))
+    await db.save_channel(1, 5, "Chan", "chan", 99)
+    ch = await db.get_channel(1)
+    for m in msgs.values():
+        if not isinstance(m, types.MessageService):
+            await saved(db, m)
+    return db, ch
+
+
+def test_the_sync_compares_and_cleans_a_channel_through_a_real_client(wire, tmp_path):
+    client, seen = wire.client, wire.seen
+    msgs, chan = synced_channel(wire)
+
+    async def go():
+        db, ch = await saved_world(tmp_path, msgs)
+        opts = SyncOptions(pause=0, grace=0)
+
+        first = await sync_channel(client, db, ch, opts)  # the first look only notes where the channel stands
+        assert (first.error, first.first_look, first.checked, first.changed) == (None, True, 4, False)
+        assert sorted(chan.msgs) == [1, 2, 3, 4, 5]
+
+        # what admins do outside the bot: delete a post, edit one, write a new one - Telegram adds notices meanwhile
+        del chan.msgs[3]
+        chan._pts += 1
+        chan.msgs[1].message = "first, edited"
+        chan._pts += 1
+        foreign(chan, "written by another admin")  # id 6
+        notice(chan, types.MessageActionChatEditTitle("A new name"))  # id 7
+        notice(chan, types.MessageActionPinMessage())  # id 8
+
+        rep = await sync_channel(client, db, ch, SyncOptions(pause=0, grace=0, clean=True))
+        assert rep.error is None and rep.anomaly is None
+        assert (rep.deleted, rep.edited, rep.adopted, rep.services_deleted, rep.services_failed) == ([3], [1], [6], 3, 0)
+        assert sorted(chan.msgs) == [1, 2, 5, 6]  # the old notice and the two new ones are gone
+
+        assert await db.find_by_message(1, 3) is None
+        assert (await db.find_by_message(1, 1)).text == "first, edited"
+        took = await db.find_by_message(1, 6)
+        assert (took.source, took.text, took.created_by, took.status) == ("adopted", "written by another admin", 0, "sent")
+
+        again = await sync_channel(client, db, ch, opts)  # nothing left to do
+        assert not again.changed and again.error is None
+        await db.close()
+
+    asyncio.run(go())
+    got = names(seen)
+    for name in ("GetFullChannelRequest", "GetMessagesRequest", "DeleteMessagesRequest"):
+        assert name in got, (name, got)
+    # the channel counter and every read name the channel the way Telegram wants it
+    fulls = [r for r in seen if isinstance(r, functions.channels.GetFullChannelRequest)]
+    assert fulls and all(isinstance(r.channel, types.InputChannel) and r.channel.channel_id == 1 for r in fulls)
+    reads = [r for r in seen if isinstance(r, functions.channels.GetMessagesRequest)]
+    assert reads and all(isinstance(r.channel, types.InputChannel) for r in reads)
+    # only the notices were deleted, in one request, and nothing was written to the channel
+    deletes = [r for r in seen if isinstance(r, functions.channels.DeleteMessagesRequest)]
+    assert [i for r in deletes for i in r.id] == [4, 7, 8]
+    assert all(isinstance(r.channel, types.InputChannel) and r.channel.channel_id == 1 for r in deletes)
+    writes = (
+        functions.messages.SendMessageRequest, functions.messages.SendMediaRequest, functions.messages.SendMultiMediaRequest,
+        functions.messages.EditMessageRequest, functions.messages.ForwardMessagesRequest, functions.messages.UpdatePinnedMessageRequest,
+    )
+    assert not [r for r in seen if isinstance(r, writes)]
+
+
+def test_the_live_look_at_a_few_messages_through_a_real_client(wire, tmp_path):
+    client, seen = wire.client, wire.seen
+    msgs, chan = synced_channel(wire)
+
+    async def go():
+        db, ch = await saved_world(tmp_path, msgs)
+        opts = SyncOptions(pause=0, grace=0)
+        await sync_channel(client, db, ch, opts)
+
+        chan.msgs[2].message = "second, edited"
+        chan._pts += 1
+        foreign(chan, "a new post")  # 6
+        note = notice(chan, types.MessageActionChatEditTitle("Another name"))  # 7
+        del chan.msgs[5]
+        chan._pts += 1
+
+        rep = await sync_ids(client, db, ch, [2, 5, 6, note.id], opts, adopt_ids=[6])
+        assert rep.error is None
+        assert (rep.edited, rep.deleted, rep.adopted, rep.services_deleted) == ([2], [5], [6], 1)
+        assert sorted(chan.msgs) == [1, 2, 3, 4, 6]
+        assert (await db.find_by_message(1, 2)).text == "second, edited"
+        assert (await db.find_by_message(1, 2)).buttons == [[{"t": "Watch", "u": "https://t.me/a/1"}]]  # the saved buttons stay
+        assert await db.find_by_message(1, 5) is None
+        await db.close()
+
+    asyncio.run(go())
+    assert [i for r in seen if isinstance(r, functions.channels.DeleteMessagesRequest) for i in r.id] == [7]
+
+
+def test_deleting_a_notice_and_the_refusals_through_a_real_client(wire, tmp_path):
+    client = wire.client
+    msgs, chan = synced_channel(wire)
+
+    async def go():
+        db, ch = await saved_world(tmp_path, msgs)
+        peer = peer_of(ch)
+
+        out = await remove(client, peer, [4])
+        assert (out.deleted, out.failed, out.error) == (1, 0, None) and 4 not in chan.msgs
+
+        chan.fail["DeleteMessagesRequest"] = rpc("ChatAdminRequiredError")
+        note = notice(chan, types.MessageActionPinMessage())
+        out = await remove(client, peer, [note.id])
+        assert (out.deleted, out.failed, out.error_name) == (0, 1, "ChatAdminRequiredError") and note.id in chan.msgs
+        chan.fail.clear()
+
+        chan.fail["GetFullChannelRequest"] = rpc("ChannelPrivateError")  # the bot lost its place in the channel
+        rep = await sync_channel(client, db, ch, SyncOptions(pause=0, grace=0))
+        assert rep.error and not rep.changed and rep.checked == 0
+        assert [p.message_id for p in await db.sent_posts(1)] == [1, 2, 3, 5]  # nothing was touched
+        await db.close()
+
+    asyncio.run(go())