Automated sync from GitHub

.gitattributes

@@ -1,8 +1,8 @@
+*.otf filter=lfs diff=lfs merge=lfs -text
+*.webp filter=lfs diff=lfs merge=lfs -text
+*.mp4 filter=lfs diff=lfs merge=lfs -text
 *.mp3 filter=lfs diff=lfs merge=lfs -text
 *.jpg filter=lfs diff=lfs merge=lfs -text
 *.jpeg filter=lfs diff=lfs merge=lfs -text
 *.png filter=lfs diff=lfs merge=lfs -text
 *.ttf filter=lfs diff=lfs merge=lfs -text
-*.otf filter=lfs diff=lfs merge=lfs -text
-*.webp filter=lfs diff=lfs merge=lfs -text
-*.mp4 filter=lfs diff=lfs merge=lfs -text

app/db.py

@@ -924,6 +924,13 @@ class Database:
                 q = q.where(Shift.dst_channel_id == dst_id)
             return (await s.execute(q.order_by(desc(Shift.created_at)).limit(1))).scalar_one_or_none()
 
+    async def last_finished_shift(self) -> Optional[Shift]:
+        """The newest finished shift that still has copies in its destination - so its Undo can be offered again."""
+        async with self.Session() as s:
+            alive = select(ShiftItem.id).where(ShiftItem.shift_id == Shift.id, ShiftItem.new_deleted.is_(False)).exists()
+            q = select(Shift).where(Shift.status == "copied", alive).order_by(desc(Shift.created_at)).limit(1)
+            return (await s.execute(q)).scalar_one_or_none()
+
     async def set_shift_status(self, sid: str, status: str) -> None:
         async with self.Session() as s:
             sh = await s.get(Shift, sid)

app/handlers/basic.py

@@ -29,8 +29,10 @@ HELP = (
     "/handleswap @old @new - change the username @old written in posts (text, captions, button names) to @new in all channels; links stay\n"
     "/undo - undo the last /replace or /handleswap\n"
     "/repost @old @new - copy every post of a channel in order (links swapped, replies, pins and links between posts kept), then delete the old posts\n"
-    "/shift @source @destination [from_id] [to_id] [-c] [-all] - copy posts from one channel into another (buttons kept, listed in My posts); "
-    "-c: links to the source's posts in all your channels are changed to the copies; -all: the destination also gets the source's name, description and photo\n"
+    "/shift @source @destination [from_id] [to_id] [-c] [-all] [--no-media] - copy posts from one channel into another (buttons kept, listed in My posts); "
+    "a channel Telegram restricts (copyright strike) is copied from My posts; "
+    "-c: links to the source's posts in all your channels are changed to the copies; -all: the destination also gets the source's name, description and photo; "
+    "--no-media: no files, only the text and buttons\n"
     "/sync - compare every channel with My posts right now (posts deleted, edited or added by others, the older posts from before the bot was added; service messages are deleted)\n"
     "/userbot - status of the helper account that deletes posts the bot is not allowed to delete\n"
     "/export - download a JSON backup of your channels and posts\n"

app/handlers/repost.py

@@ -147,23 +147,45 @@ def plan_text(p: RepostPlan, o: RepostOptions, title: str, delay: float) -> str:
 
 
 def held_plan_lines(p: RepostPlan, title: str, *, shift: bool = False) -> list:
-    """Plan lines about posts Telegram holds back (it shows a notice, for example for a copyright strike, instead of them)."""
+    """Plan lines about posts Telegram restricts (a copyright strike: the channel is "unavailable", or its posts are
+    marked or shown as a notice)."""
     lines = []
     name = esc(title)
-    if p.channel_restricted:
-        lines.append(f"• ⚠️ Telegram restricts {name} itself: “{esc(p.channel_restricted)}”")
     said = f" (“{esc(p.held_why)}”)" if p.held_why else ""
-    if p.held_saved:
+    if p.channel_restricted and shift:
+        lines.append(f"• ⚠️ Telegram restricts {name} itself: “{esc(p.channel_restricted)}”")
+    elif p.channel_restricted:
+        lines.append(
+            f"• ⚠️ Telegram restricts {name} itself: “{esc(p.channel_restricted)}”. Telegram may hide what is posted into "
+            "such a channel as well, so a repost here may not bring the posts back. To save a struck channel use "
+            "/shift: it copies the posts from My posts into another channel."
+        )
+    if p.held_saved and p.no_media:
         lines.append(
-            f"• 📋 Telegram holds back {p.held_saved} post(s) of {name}{said} - it shows a notice instead of them. "
-            "Those are copied from what My posts saved of them: text, formatting, buttons and media (a file that "
+            f"• 📋 {p.held_saved} post(s) of {name} are restricted by Telegram{said}. They are copied from what My posts "
+            "saved of them - text, formatting and buttons - not from what Telegram shows now."
+        )
+    elif p.held_saved:
+        lines.append(
+            f"• 📋 {p.held_saved} post(s) of {name} are restricted by Telegram{said}. They are copied from what My posts "
+            "saved of them - text, formatting, buttons and media - not from what Telegram shows now (a file that "
             "Telegram refuses goes out without it)."
         )
+    if p.live_restricted and shift:
+        lines.append(
+            f"• ⚠️ {p.live_restricted} restricted post(s) of {name} have no saved copy in My posts, but Telegram still "
+            "hands out their content, so they are copied as they are now."
+        )
+    elif p.live_restricted:
+        lines.append(
+            f"• ⚠️ Telegram marks {p.live_restricted} post(s) of {name} as restricted but still hands out their content, "
+            "so they are copied as they are now."
+        )
     if p.held_skipped and shift:
         lines.append(
-            f"• ⚠️ Telegram holds back {p.held_skipped} more post(s) of {name}{said} and My posts has no saved copy of "
-            "them, so they are skipped and the shift will show as not complete. If you have a backup made before that, "
-            "put it back with /import and run /shift again."
+            f"• ⚠️ Telegram shows only its notice for {p.held_skipped} more post(s) of {name}{said} and My posts has no "
+            "saved copy of them, so they are skipped and the shift will show as not complete. If you have a backup made "
+            "before that, put it back with /import and run /shift again."
         )
     elif p.held_skipped:
         lines.append(
@@ -171,36 +193,49 @@ def held_plan_lines(p: RepostPlan, title: str, *, shift: bool = False) -> list:
             "Copying the notice would be wrong, so they are skipped, and the old posts can't be deleted afterwards "
             "(the repost will show as not complete)."
         )
+    if p.channel_restricted or p.held_saved or p.live_restricted:
+        lines.append("• The new posts are read back: if Telegram itself hides them, the report says so.")
     return lines
 
 
 def held_notes(res) -> list:
-    """Lines about the posts Telegram holds back, for the end of a repost or shift."""
+    """Lines about the posts Telegram restricts, for the end of a repost or shift."""
     notes = []
 
     def sample(ids, n=8):
         return ", ".join(f"#{i}" for i in ids[:n]) + ("…" if len(ids) > n else "")
 
+    what = "text, formatting and buttons" if getattr(res, "no_media", False) else "text, formatting, buttons and media"
     if res.from_db:
         if res.from_saved:
             notes.append(
                 f"📋 The source can't be read, so {res.from_saved} post(s) were copied from what My posts saved of them: "
-                "text, formatting, buttons and media - without replies, pins and albums."
+                f"{what} - without replies, pins and albums."
             )
     elif res.from_saved:
         notes.append(
-            f"📋 {res.from_saved} post(s) are held back by Telegram - it shows a notice instead of them. They were "
-            "copied from what My posts saved of them: text, formatting, buttons and media."
+            f"📋 {res.from_saved} post(s) are restricted by Telegram. They were copied from what My posts saved of "
+            f"them: {what}."
         )
     if res.media_lost:
         notes.append(
             f"⚠️ {len(res.media_lost)} of them went out without their media, because Telegram refuses the saved file "
             f"({sample(res.media_lost)})."
         )
+    if res.live_restricted and res.used_saved:
+        notes.append(
+            f"⚠️ {res.live_restricted} restricted post(s) had no saved copy in My posts, so they were copied as Telegram "
+            "hands them out (their content was intact)."
+        )
+    elif res.live_restricted:
+        notes.append(
+            f"⚠️ Telegram marks {res.live_restricted} post(s) as restricted but still handed out their content, so they "
+            "were copied as they are."
+        )
     if res.held_back and res.used_saved:
         notes.append(
-            f"⚠️ {len(res.held_back)} post(s) are held back by Telegram and My posts has no saved copy of them, so they "
-            f"were not copied ({sample(res.held_back)}). Put a backup made before that back with /import, then use "
+            f"⚠️ {len(res.held_back)} post(s): Telegram shows only its notice and My posts has no saved copy of them, so "
+            f"they were not copied ({sample(res.held_back)}). Put a backup made before that back with /import, then use "
             "“Try the rest again”."
         )
     elif res.held_back:
@@ -212,9 +247,70 @@ def held_notes(res) -> list:
     return notes
 
 
-def finishing_notes(res) -> list:
-    """Lines about answers, links and pins for the end of a repost."""
-    notes = held_notes(res)
+def no_media_notes(res) -> list:
+    """Lines about the files that /shift --no-media left out."""
+    if not getattr(res, "no_media", False):
+        return []
+    notes = []
+    if res.files_left_out:
+        notes.append(f"🚫 {res.files_left_out} post(s) were copied as text only - their file was left out (--no-media).")
+    if res.file_only:
+        ids = ", ".join(f"#{i}" for i in res.file_only[:8]) + ("…" if len(res.file_only) > 8 else "")
+        notes.append(f"🚫 {len(res.file_only)} post(s) are only a file, so there was nothing to copy of them ({ids}).")
+    return notes
+
+
+def hidden_notes(res, *, shift: bool = False) -> list:
+    """Lines about new posts that TELEGRAM itself hides (they were read back after posting)."""
+    if not getattr(res, "hidden", None):
+        return []
+    k, n = len(res.hidden), res.checked
+    with_file = sum(1 for _, has in res.hidden if has)
+    plain = k - with_file
+    why = f" (“{esc(res.hidden_why)}”)" if res.hidden_why else ""
+    lines = [
+        f"⚠️ <b>Telegram itself hides {k} of the {n} new post(s) I read back</b>{why}: right after they were created they "
+        "carry its restriction mark, so its apps show the notice instead of them. The bot did not copy a notice - it "
+        "never posts one: Telegram marks these posts itself, because of something in them or because of the destination."
+    ]
+    if with_file:
+        hint = (
+            " <code>/shift … --no-media</code> copies the text and buttons without the files."
+            if shift
+            else ""
+        )
+        lines.append(
+            f"• {with_file} post(s) of these carry a file. Telegram may hide a message that carries a file it has "
+            f"blocked for copyright, wherever it is posted.{hint}"
+        )
+    if plain:
+        lines.append(
+            f"• {plain} post(s) of these have no file: Telegram hides them for their text or links, or because it "
+            "restricts the new channel too."
+        )
+    if getattr(res, "dst_restricted", None):
+        lines.append(f"• Telegram restricts the destination channel itself: “{esc(res.dst_restricted)}”.")
+    if getattr(res, "hidden_stop", False):
+        lines.append(
+            f"⏹ <b>I stopped after {res.copied_units} post(s)</b>: Telegram hides every new copy I read back, so going on "
+            "would only add more posts nobody can see. Nothing else was posted. ▶️ <b>Continue anyway</b> copies the rest; "
+            "↩️ <b>Undo</b> removes the copies."
+        )
+    lines.append("The saved posts in My posts are intact. Undo removes the new copies again.")
+    return lines
+
+
+def hidden_progress(res) -> str:
+    """One line for the running progress message when Telegram hides what was just posted."""
+    if not getattr(res, "hidden", None):
+        return ""
+    why = f" (“{esc(res.hidden_why)}”)" if res.hidden_why else ""
+    return f"\n⚠️ Telegram itself hides {len(res.hidden)} of the {res.checked} new posts checked so far{why}. ⏹ Stop is below."
+
+
+def finishing_notes(res, *, shift: bool = False) -> list:
+    """Lines about answers, links and pins for the end of a repost or shift."""
+    notes = held_notes(res) + no_media_notes(res) + hidden_notes(res, shift=shift)
     if res.replies:
         notes.append(f"↩️ {res.replies} post(s) answer the copy of the post they answered.")
     if res.reply_lost:
@@ -430,6 +526,7 @@ def register(ctx: Ctx) -> None:
                 body = f"🔗 Copied {done}{of} posts. Now pointing links at the new copies and pinning…"
             else:
                 body = f"⏳ Copying <b>{esc(ch.title)}</b>: {done}{of} posts…\nThe old posts stay untouched."
+                body += hidden_progress(res)
             try:
                 await edit_callback_message(event, body, [[Button.inline("⏹ Stop", f"rps:{mig.id}")]])
             except errors.MessageNotModifiedError:

app/handlers/shift.py

@@ -18,7 +18,7 @@ from ..repost_engine import RepostOptions, RepostPlan, RepostResult, plan_repost
 from ..shift_engine import delete_shift_copies, plan_from_my_posts, run_shift, shift_profile, source_of
 from ..tgutil import esc, explain_rpc, get_rights, post_link
 from ..userbot import delete_problem_text
-from .repost import finishing_notes, held_plan_lines
+from .repost import finishing_notes, held_plan_lines, hidden_progress
 
 log = logging.getLogger(__name__)
 
@@ -31,23 +31,28 @@ USAGE = (
     "<code>/shift @source @destination</code> - every post\n"
     "<code>/shift @source @destination 15 20</code> - only the posts with message ids 15 to 20\n"
     "<code>/shift @source @destination 15</code> - only post 15\n\n"
-    "A post that Telegram holds back in the source (a copyright strike: it shows “This message couldn't be displayed on "
-    "your device due to copyright infringement” instead of the post) is copied from what My posts saved of it. A post "
-    "without a saved copy is skipped.\n\n"
-    "Two extras, each optional, in any order, with or without a range:\n"
+    "A channel that Telegram restricts (a copyright strike: “This channel is unavailable due to copyright "
+    "infringement”, its posts shown as “This message couldn't be displayed on your device due to copyright "
+    "infringement”) is copied from what My posts saved of it - every post that has a saved copy, however intact "
+    "Telegram's version looks. Telegram's notice is never copied. A post without a saved copy is copied as Telegram "
+    "hands it out if it is still the post itself, and skipped if only the notice is left.\n\n"
+    "Three extras, each optional, in any order, with or without a range:\n"
     "<code>-c</code> - afterwards every link to a post of the source - in ANY of your connected channels - is pointed "
     "at the copy in the destination\n"
     "<code>-all</code> - afterwards the destination gets the source's name, description and profile photo\n"
-    "<code>/shift @source @destination -c -all</code> does both."
+    "<code>--no-media</code> - no file is copied: every post goes out as its text with its formatting and buttons "
+    "(use it when Telegram hides the copies because of their files)\n"
+    "<code>/shift @source @destination -c -all</code> does the first two."
 )
 
 
 @dataclass
 class ShiftFlags:
-    """The two optional extras of /shift."""
+    """The optional extras of /shift."""
 
     links: bool = False  # -c: links to the source's posts, in all connected channels, follow the copies
     profile: bool = False  # -all: the destination gets the source's name, description and profile photo
+    no_media: bool = False  # --no-media: the copies get no files (text, formatting and buttons only)
 
 
 @dataclass
@@ -66,26 +71,30 @@ class ShiftJob:
     created: float = field(default_factory=time.monotonic)
 
 
-_OPTION = re.compile(r"^-{1,2}([A-Za-z][A-Za-z0-9_]*)$")
+_OPTION = re.compile(r"^[-\u2013\u2014]{1,2}([A-Za-z][A-Za-z0-9_-]*)$")  # phone keyboards turn "--" into a long dash
 
 
 def parse_shift_args(raw: str) -> tuple:
     """-> (source text, destination text, first id or None, last id or None, ShiftFlags).
 
-    -c and -all can stand anywhere, in any order ("- all" with a space is understood too); a channel id such as
-    -1001234567890 is not an option."""
-    text = re.sub(r"(?<!\S)(-{1,2})\s+(all|c)(?!\S)", r"\1\2", raw or "", flags=re.I)
+    -c, -all and --no-media can stand anywhere, in any order ("- all" with a space is understood too, and so are
+    -nomedia and a long dash in place of --); a channel id such as -1001234567890 is not an option."""
+    text = re.sub(r"(?<!\S)([-\u2013\u2014]{1,2})\s+(all|c|no[-_]?media)(?!\S)", r"\1\2", raw or "", flags=re.I)
     flags, toks = ShiftFlags(), []
     for t in text.split():
         m = _OPTION.match(t)
         if m is None:
             toks.append(t)
-        elif m.group(1).lower() == "c":
+            continue
+        name = re.sub(r"[-_]", "", m.group(1)).lower()
+        if name == "c":
             flags.links = True
-        elif m.group(1).lower() == "all":
+        elif name == "all":
             flags.profile = True
+        elif name == "nomedia":
+            flags.no_media = True
         else:
-            raise ValueError(f"Unknown option {t}. The options are -c and -all.")
+            raise ValueError(f"Unknown option {t}. The options are -c, -all and --no-media.")
     if len(toks) < 2 or len(toks) > 4:
         raise ValueError("Give the source and the destination channel, and optionally the first and the last message id.")
     nums: list = []
@@ -122,6 +131,13 @@ def plan_text(
     if p.dropped_posts:
         lines.append(f"• {p.dropped_posts} post(s) carry other bots' buttons (reactions...) - those are not copied")
     lines += held_plan_lines(p, src.title, shift=True)
+    if p.no_media:
+        note = "• 🚫 <b>--no-media</b>: no file is copied - every post goes out as its text with its formatting and buttons"
+        if p.files_left_out:
+            note += f". {p.files_left_out} post(s) carry a file that stays out"
+        if p.file_only:
+            note += f"; {p.file_only} post(s) are only a file, so there is nothing to copy of them and they are skipped"
+        lines.append(note + ".")
     if p.post_links:
         lines.append(
             f"• {p.post_links} link(s) in {p.post_link_posts} post(s) point at other posts of {esc(src.title)}: those that "
@@ -160,13 +176,30 @@ def plan_text(
             )
     minutes = max(1, round(p.eta_seconds(delay) / 60))
     if p.from_db:
+        what = "its text, formatting and buttons" if p.no_media else (
+            "its text, formatting, buttons and media (a file that Telegram refuses goes out without it)"
+        )
         how = (
-            "<b>How:</b> every post is posted again from what My posts saved of it - its text, formatting, buttons and "
-            "media (a file that Telegram refuses goes out without it) - and every button is read back to make sure it "
-            "is there. Copies go to the end of the destination silently, in the original order, and are registered in "
-            "My posts."
+            f"<b>How:</b> every post is posted again from what My posts saved of it - {what} - and every button is read "
+            "back to make sure it is there. Copies go to the end of the destination silently, in the original order, and "
+            "are registered in My posts."
         )
         kept = "<b>Not kept:</b> original dates, view counts, reactions, comments, poll votes, pins, replies and albums (every picture of an album becomes a post of its own)."
+    elif p.no_media:
+        how = (
+            "<b>How:</b> every post is posted again as its text with its formatting and buttons - no file is copied - and "
+            "every button is read back to make sure it is there. Copies go to the end of the destination silently, in the "
+            "original order, and are registered in My posts."
+        )
+        kept = "<b>Not kept:</b> the files, albums (a caption becomes a post of its own), original dates, view counts, reactions, comments, poll votes and pins."
+    elif p.channel_restricted:
+        how = (
+            "<b>How:</b> Telegram copies nothing out of a restricted channel, so every post is posted again - from what "
+            "My posts saved of it, or as Telegram hands it out when there is no saved copy - with its text, formatting, "
+            "media and buttons in one go, and every button is read back to make sure it is there. Copies go to the end of "
+            "the destination silently, in the original order, and are registered in My posts."
+        )
+        kept = "<b>Not kept:</b> original dates, view counts, reactions, comments, poll votes and pins."
     else:
         how = (
             "<b>How:</b> posts are copied by Telegram itself where possible (exactly as they are); posts with buttons or "
@@ -174,6 +207,8 @@ def plan_text(
             "back to make sure it is there. Copies go to the end of the destination silently, in the original order, and "
             "are registered in My posts."
         )
+        if p.held_saved or p.live_restricted:
+            how += " Posts that Telegram restricts are never copied by Telegram itself: they are posted again."
         kept = "<b>Not kept:</b> original dates, view counts, reactions, comments, poll votes and pins."
     lines += [
         "",
@@ -201,6 +236,8 @@ def register(ctx: Ctx) -> None:
             first_copy = f'\n<a href="{post_link(dst, c["first_new"])}">First copy</a>'
         undo = [Button.inline("↩️ Undo - remove the copies", f"sfu:{sh.id}")]
         marks = await db.marks_of(sh.id)
+        if "nomedia" in marks:
+            route += " (without files)"
         extras = []
         if "relink" in marks:
             extras.append([Button.inline("🔗 Update links again", f"sfl:{sh.id}")])
@@ -218,7 +255,8 @@ def register(ctx: Ctx) -> None:
             kb = []
         elif sh.status == "stopped":
             text = f"⏹ {route}: <b>stopped</b>. {c['copied']} message(s) copied so far.{first_copy}"
-            kb = [[Button.inline("▶️ Continue", f"sfc:{sh.id}")], undo]
+            go = "▶️ Continue anyway" if "hiddenok" in marks else "▶️ Continue"
+            kb = [[Button.inline(go, f"sfc:{sh.id}")], undo]
         elif sh.status == "copying":
             text = f"⏳ {route}: a shift was running when the bot stopped. {c['copied']} message(s) copied.{first_copy}"
             kb = [[Button.inline("▶️ Continue", f"sfc:{sh.id}")], undo]
@@ -241,8 +279,13 @@ def register(ctx: Ctx) -> None:
             if sh is not None:
                 text, kb = await view_for(sh)
                 await say(event, "Unfinished shift:\n\n" + text, kb)
-            else:
-                await say(event, USAGE)
+                return
+            await say(event, USAGE)
+            # a finished shift keeps its Undo: offer it again, so it can be found without scrolling up to its old message
+            last = await db.last_finished_shift()
+            if last is not None:
+                text, kb = await view_for(last)
+                await say(event, "Latest finished shift - you can still undo it:\n\n" + text, kb)
             return
         try:
             src_text, dst_text, first, last, flags = parse_shift_args(raw)
@@ -278,12 +321,12 @@ def register(ctx: Ctx) -> None:
             async with ctx.lock:
                 plan = await plan_repost(
                     client, src, RepostOptions(), progress=prog, first_id=first, last_id=last,
-                    saved=saved_lookup(db, src.id), use_saved=True,
+                    saved=saved_lookup(db, src.id), use_saved=True, no_media=flags.no_media,
                 )
         except errors.RPCError as e:
             why = explain_rpc(e)
             # a channel that can't be read any more (banned, private ...) may still be in My posts: copy it from there
-            plan = await plan_from_my_posts(db, src, first, last, unreadable=why)
+            plan = await plan_from_my_posts(db, src, first, last, unreadable=why, no_media=flags.no_media)
             if plan is None:
                 raise UserError(
                     f"I can't read {src.title}: {why}\nThe bot has to be a member of the source channel "
@@ -336,6 +379,7 @@ def register(ctx: Ctx) -> None:
                 body = f"🔗 Copied {done}{of} posts. Now pointing links at the new copies…"
             else:
                 body = f"⏳ Shifting {names}: {done}{of} posts…\nThe source stays untouched."
+                body += hidden_progress(res)
             try:
                 await edit_callback_message(event, body, [[Button.inline("⏹ Stop", f"sfs:{sid}")]])
             except errors.MessageNotModifiedError:
@@ -353,6 +397,8 @@ def register(ctx: Ctx) -> None:
             job.running = False
         status = "stopped" if res.stopped else ("incomplete" if (res.failed or res.aborted or res.held_back) else "copied")
         await db.set_shift_status(sid, status)
+        if res.hidden_stop:  # stopped because Telegram hides the copies: pressing Continue means "go on anyway"
+            await db.set_mark(sid, "hiddenok")
         sh = await db.get_shift(sid)
         text, kb = await view_for(sh)
         notes = []
@@ -372,7 +418,7 @@ def register(ctx: Ctx) -> None:
                 notes.append("ReplyParentMissing: that post answers a post whose copy failed; it follows when you try again.")
         if res.aborted:
             notes.append(f"Stopped early because of {esc(res.aborted)}.")
-        notes += finishing_notes(res)
+        notes += finishing_notes(res, shift=True)
         await edit_callback_message(event, text + ("\n\n" + "\n".join(notes) if notes else ""), kb)
 
     @on_cb(ctx, "sfa")
@@ -397,6 +443,8 @@ def register(ctx: Ctx) -> None:
             await db.set_mark(sid, "relink")
         if job.flags.profile:
             await db.set_mark(sid, "profile")
+        if job.flags.no_media:  # continuing the shift later goes on without files as well
+            await db.set_mark(sid, "nomedia")
         if p.from_db:  # the source can't be read: continuing the shift goes on from My posts as well
             await db.set_mark(sid, "fromdb")
         ctx.pending.pop(parts[0], None)

app/repost_engine.py

@@ -17,10 +17,14 @@ What a plain copy would lose is put back as well:
     copy already exists, afterwards (finalize_repost) for posts that are copied later;
   * a post that was pinned has its copy pinned at the end.
 
-A post that TELEGRAM holds back (a copyright strike shows "This message couldn't be displayed on your device due to
-copyright infringement" instead of the post) is never copied as it looks now - that would copy the notice. /shift
-rebuilds such a post from what My posts saved of it (text, formatting, buttons, media); /repost leaves it alone.
-A post that has no saved copy is skipped and reported.
+A post that TELEGRAM restricts (a copyright strike: the whole channel is "unavailable due to copyright infringement",
+or the post carries a restriction mark, or Telegram's apps show "This message couldn't be displayed on your device due
+to copyright infringement" instead of the post) is not copied as Telegram hands it out now. /shift rebuilds such a post
+from what My posts saved of it (text, formatting, buttons, media) - for a restricted CHANNEL every post that has a saved
+copy, however intact Telegram's version looks. A post without a saved copy is copied as Telegram hands it out when that
+is the post itself, and skipped (and reported) when it is only Telegram's notice. Telegram's notice is never posted, from
+whichever source. /repost leaves a post whose content Telegram holds back alone.
+Every copy of a restricted source is read back: if Telegram itself hides the new post, that is reported (copy_loop).
 """
 from __future__ import annotations
 
@@ -37,7 +41,7 @@ from .crosslinks import CrossResult, relink_after_repost
 from .linkswap import compute_message_changes, url_only_markup
 from .postlinks import PostLinks, relink_message
 from .replace_engine import CHUNK, FALLBACK_EMPTY_BATCHES, fetch_messages, is_skippable, preview_flag
-from .restrictions import is_placeholder_text, message_restriction, norm_text, probe_channel
+from .restrictions import PLACEHOLDER_NOTE, is_placeholder_text, message_restriction, probe_channel
 from .tgutil import (
     FATAL,
     build_markup,
@@ -178,24 +182,39 @@ def saved_lookup(db, channel_id: int) -> SavedLookup:
     return lookup
 
 
-def held_back_reason(unit: list, rows: dict) -> Optional[str]:
-    """What Telegram says when it holds this post back (a copyright strike ...), else None.
-
-    "Held back" means that what Telegram hands out is its notice - or nothing - instead of the post: copying that would
-    copy the notice. A post that carries Telegram's restriction mark but is handed out as it is (its text, its media)
-    is an ordinary post for a copy, and so is a text that the saved copy has too (the owner's own wording)."""
+def usable_row(row) -> bool:
+    """A saved post that can take the place of a restricted post: it has something to post, and what it says is not
+    Telegram's notice (a saved copy that an older version made from a restricted post can hold the notice)."""
+    if row is None:
+        return False
+    text = row.text or ""
+    if is_placeholder_text(text):
+        return False
+    return bool(text.strip() or row.media_file_id)
+
+
+def copy_source(unit: list, rows: dict, *, restricted: Optional[str] = None, use_saved: bool = False) -> tuple:
+    """Where the copy of this post (or album) comes from -> (source, why).
+
+      "live"  - as Telegram hands the post out: nothing restricts it, or Telegram restricts it but still hands out the
+                post itself and there is no saved copy to use instead;
+      "saved" - from what My posts saved of it: Telegram restricts the channel (`restricted`) or the post (a copyright
+                strike) and My posts has every part of it. Only when `use_saved` (/shift) - the saved copy wins even when
+                Telegram's version still looks intact, because that is the owner's rule for a struck channel;
+      "skip"  - not copied: what Telegram hands out is its notice - or nothing - and My posts can't take over.
+
+    `why` is Telegram's own wording about the restriction, None when nothing restricts the post."""
+    marked = next((w for w in (message_restriction(m) for m in unit) if w), None)
+    why = marked or restricted
+    if why is None:
+        return "live", None
+    if use_saved and all(usable_row(rows.get(m.id)) for m in unit):
+        return "saved", why
     for m in unit:
-        row = rows.get(m.id)
-        saved_text = row.text if row is not None else None
-        why = message_restriction(m, saved_text)
-        if not why:
-            continue
         live = m.message or ""
-        notice = is_placeholder_text(live) and (saved_text is None or norm_text(live) != norm_text(saved_text))
-        nothing = not live.strip() and getattr(m, "media", None) is None
-        if notice or nothing:
-            return why
-    return None
+        if is_placeholder_text(live) or (not live.strip() and getattr(m, "media", None) is None):
+            return "skip", why
+    return "live", why
 
 
 def stand_in(m, row):
@@ -219,17 +238,47 @@ def stand_in(m, row):
 
 
 def stand_ins(unit: list, rows: dict) -> Optional[list]:
-    """The saved copies that stand in for a held-back post, or None when My posts does not have all of them (or one
-    of them has nothing to post)."""
+    """The saved copies that stand in for a restricted post, or None when My posts does not have all of them (or one of
+    them has nothing to post, or holds only Telegram's notice)."""
     out = []
     for m in unit:
         row = rows.get(m.id)
-        if row is None or not ((row.text or "").strip() or row.media_file_id):
+        if not usable_row(row):
             return None
         out.append(stand_in(m, row))
     return out
 
 
+FILE_MEDIA = (types.MessageMediaPhoto, types.MessageMediaDocument)  # a photo, video, document, voice, sticker ...
+
+
+def has_file(m) -> bool:
+    """The post carries a file. A link preview, a poll, a location or a contact is not a file."""
+    return isinstance(getattr(m, "media", None), FILE_MEDIA)
+
+
+def without_files(unit: list) -> tuple:
+    """/shift --no-media: the posts of `unit` as they look without their files -> (those that still have a text, the
+    number of files left out). A post that was only a file has nothing left to post and is dropped."""
+    out, lost = [], 0
+    for m in unit:
+        c = m
+        if has_file(m):
+            c = copy.copy(m)
+            c.media, c.saved_ref, c.invert_media = None, None, None
+            lost += 1
+        if (c.message or "").strip():
+            out.append(c)
+    return out, lost
+
+
+async def single_posts(units: AsyncIterator[list]) -> AsyncIterator[list]:
+    """/shift --no-media: an album can't be rebuilt without its pictures, so every message of it is a post of its own."""
+    async for unit in units:
+        for m in unit:
+            yield [m]
+
+
 def saved_message(row, channel_id: int):
     """A post built from its saved copy alone, for a channel that can't be read at all. Only the saved parts exist: no
     album, no reply, no pin."""
@@ -240,13 +289,14 @@ def saved_message(row, channel_id: int):
 
 
 def saved_rows_between(rows: list, first: Optional[int], last: Optional[int]) -> list:
-    """The saved posts (oldest first, one per message id) that have something to post, between two message ids."""
+    """The saved posts (oldest first, one per message id) that can be posted - they have something to post that is not
+    Telegram's notice - between two message ids."""
     out, seen = [], set()
     for row in sorted(rows, key=lambda r: (r.message_id or 0, r.id)):
         mid = row.message_id
         if mid is None or mid in seen or (first and mid < first) or (last and mid > last):
             continue
-        if (row.text or "").strip() or row.media_file_id:
+        if usable_row(row):
             seen.add(mid)
             out.append(row)
     return out
@@ -282,8 +332,12 @@ class RepostPlan:
     post_links: int = 0
     pinned: int = 0  # pinned posts (their copies get pinned)
     replies: int = 0  # posts that answer another post
-    held_saved: int = 0  # posts Telegram holds back that are copied from what My posts saved of them (counted in units)
-    held_skipped: int = 0  # posts Telegram holds back that can't be copied (not counted in units)
+    held_saved: int = 0  # restricted posts that are copied from what My posts saved of them (counted in units)
+    held_skipped: int = 0  # restricted posts that can't be copied: only Telegram's notice is left (not counted in units)
+    live_restricted: int = 0  # restricted posts without a saved copy that are copied as Telegram hands them out
+    no_media: bool = False  # /shift --no-media: the files are left out
+    files_left_out: int = 0  # ... posts that are copied as text without their file (counted in units)
+    file_only: int = 0  # ... posts that are only a file: nothing to copy (not counted in units)
     held_why: Optional[str] = None  # what Telegram says about them
     channel_restricted: Optional[str] = None  # what Telegram says when it restricts the whole channel
     from_db: bool = False  # the channel can't be read: the plan (and the copy) is made from My posts alone
@@ -324,20 +378,34 @@ def tally(plan: RepostPlan, m, opts: RepostOptions, links: PostLinks) -> None:
         plan.replies += 1
 
 
-def plan_from_saved(ch, rows: list, opts: RepostOptions, *, first_id=None, last_id=None, unreadable=None) -> RepostPlan:
+def plan_from_saved(
+    ch, rows: list, opts: RepostOptions, *, first_id=None, last_id=None, unreadable=None, no_media: bool = False
+) -> RepostPlan:
     """The plan of a copy made from My posts alone (the channel can't be read): one post per saved message."""
-    plan = RepostPlan(channel=ch, partial=bool(first_id or last_id), from_db=True, unreadable=unreadable)
+    plan = RepostPlan(
+        channel=ch, partial=bool(first_id or last_id), from_db=True, unreadable=unreadable, no_media=no_media
+    )
     links = PostLinks.for_channel(ch)
     chosen = saved_rows_between(rows, first_id, last_id)
     for row in chosen:
+        m = saved_message(row, ch.id)
+        if no_media:
+            kept, lost = without_files([m])
+            if not kept:
+                plan.file_only += 1
+                continue
+            m, plan.files_left_out = kept[0], plan.files_left_out + lost
         plan.units += 1
-        tally(plan, saved_message(row, ch.id), opts, links)
+        tally(plan, m, opts, links)
     if chosen:
         plan.first, plan.last = chosen[0].message_id, chosen[-1].message_id
         plan.scanned = plan.last
     else:
         plan.first, plan.last = first_id or 1, last_id or 0
+    if not plan.units:
         plan.error = "There is nothing to copy."
+        if plan.file_only:
+            plan.error = "Every saved post in this range is only a file, so without files there is nothing to copy."
     return plan
 
 
@@ -366,12 +434,14 @@ async def plan_repost(
     last_id: Optional[int] = None,
     saved: Optional[SavedLookup] = None,
     use_saved: bool = False,
+    no_media: bool = False,
 ) -> RepostPlan:
     """Read the channel and count what a copy would do. first_id / last_id limit the range (used by /shift).
     `saved` looks up My posts' copies of the channel's posts; with `use_saved` a post Telegram holds back is counted as
-    the saved post it will be copied from (/shift), without it such a post is left out (/repost)."""
+    the saved post it will be copied from (/shift), without it such a post is left out (/repost). With `no_media` the
+    files are left out of the copies (/shift --no-media): a post that is only a file has nothing left to copy."""
     peer = peer_of(ch)
-    plan = RepostPlan(channel=ch, partial=bool(opts.last))
+    plan = RepostPlan(channel=ch, partial=bool(opts.last), no_media=no_media)
     try:
         top, plan.channel_restricted, _ = await probe_channel(client, peer)
     except Exception as e:  # the counter is only a help: without it the ids are read until nothing more turns up
@@ -411,20 +481,34 @@ async def plan_repost(
                 plan.service += 1
                 continue
             gid = getattr(m, "grouped_id", None)
-            new_unit = gid is None or gid != prev_gid
+            new_unit = no_media or gid is None or gid != prev_gid  # without files an album is a post per message
             row = rows.get(m.id)
-            why = held_back_reason([m], rows)
+            # a struck channel changes how /shift copies (My posts first); /repost copies as before, post by post
+            source, why = copy_source(
+                [m], rows, restricted=plan.channel_restricted if use_saved else None, use_saved=use_saved
+            )
             if why:
                 plan.held_why = plan.held_why or why
-                if not (use_saved and stand_ins([m], rows) is not None):
-                    plan.held_skipped += 1 if new_unit else 0  # nothing to copy it from
+            if source == "skip":
+                plan.held_skipped += 1 if new_unit else 0  # nothing to copy it from
+                prev_gid = gid
+                continue
+            if source == "saved":
+                m = stand_in(m, row)  # counted as the saved post it will be copied from
+            if no_media:
+                kept, lost = without_files([m])
+                if not kept:  # nothing but a file: nothing to copy without it
+                    plan.file_only += 1
                     prev_gid = gid
                     continue
+                m, plan.files_left_out = kept[0], plan.files_left_out + lost
+            if source == "saved":
                 plan.held_saved += 1 if new_unit else 0
-                m = stand_in(m, row)  # counted as the saved post it will be copied from
+            elif why:
+                plan.live_restricted += 1 if new_unit else 0
             if new_unit:
                 plan.units += 1
-                if gid is not None:
+                if gid is not None and not no_media:
                     plan.albums += 1
             prev_gid = gid
             tally(plan, m, opts, links)
@@ -444,6 +528,8 @@ async def plan_repost(
                 "Telegram holds back every post in this range (it shows a notice instead of the post) and My posts has "
                 "no saved copy of them, so there is nothing to copy."
             )
+        elif plan.file_only:
+            plan.error = "Every post in this range is only a file, so without files there is nothing to copy."
     return plan
 
 
@@ -626,9 +712,11 @@ async def copy_unit(
     dst_peer=None,
     *,
     from_saved: bool = False,
+    no_forward: bool = False,
 ) -> list:
     """Create the copy of one post (or album) at the end of `dst_peer` (default: the same channel `peer`).
-    `from_saved`: the posts in `unit` are saved copies standing in for posts Telegram holds back - never forwarded."""
+    `from_saved`: the posts in `unit` are saved copies standing in for posts Telegram restricts - never forwarded.
+    `no_forward`: Telegram restricts these posts and refuses to copy them itself: they are rebuilt from their parts."""
     dst = dst_peer if dst_peer is not None else peer
     if from_saved:
         news = await resend_saved(client, dst, unit, finals, st, reply_to)
@@ -638,7 +726,7 @@ async def copy_unit(
             st.replies += 1
         return news
     pure = all(is_pure(m, f, reply_to) for m, f in zip(unit, finals))
-    if st.mode == "forward" and pure:
+    if st.mode == "forward" and pure and not no_forward:
         try:
             news = await forward_unit(client, peer, unit, dst)
             if len(news) == len(unit):
@@ -659,7 +747,7 @@ async def copy_unit(
         # rebuild necessary, a plain copy by Telegram still works - without the reply.
         if isinstance(e, errors.RPCError) and type(e).__name__ in FATAL:
             raise
-        if reply_to and st.mode == "forward" and all(is_pure(m, f) for m, f in zip(unit, finals)):
+        if reply_to and st.mode == "forward" and not no_forward and all(is_pure(m, f) for m, f in zip(unit, finals)):
             news = await forward_unit(client, peer, unit, dst)
             if len(news) != len(unit):
                 await delete_ids(client, dst, [n.id for n in news])
@@ -721,12 +809,21 @@ class RepostResult:
     replies: int = 0  # copies created as an answer to the copy of the post the original answered
     reply_lost: int = 0  # answers that could only be copied without the reply
     reply_dropped: int = 0  # answers to a post that no longer exists (nothing to answer)
-    from_saved: int = 0  # posts Telegram holds back that were copied from their saved copy in My posts
+    from_saved: int = 0  # restricted posts that were copied from their saved copy in My posts
     media_lost: list = field(default_factory=list)  # ... of which these (old ids) went out without their media
-    held_back: list = field(default_factory=list)  # posts Telegram holds back that were not copied (first old id)
-    held_why: Optional[str] = None  # what Telegram says about them
-    used_saved: bool = False  # the run could take held-back posts from My posts (/shift)
+    held_back: list = field(default_factory=list)  # posts that were not copied: only Telegram's notice was left (old id)
+    held_why: Optional[str] = None  # what Telegram says about the restricted posts
+    live_restricted: int = 0  # restricted posts without a saved copy, copied as Telegram hands them out
+    no_media: bool = False  # /shift --no-media: no file was copied
+    files_left_out: int = 0  # ... posts that were copied as text, without their file
+    file_only: list = field(default_factory=list)  # ... posts that are only a file: nothing to copy (old id)
+    used_saved: bool = False  # the run could take restricted posts from My posts (/shift)
     from_db: bool = False  # the channel can't be read: every post was copied from My posts alone
+    checked: int = 0  # new posts that were read back to see whether Telegram itself hides them
+    hidden: list = field(default_factory=list)  # [(old id, had media)] new posts Telegram itself hides (restriction mark)
+    hidden_why: Optional[str] = None  # what Telegram says about them
+    dst_restricted: Optional[str] = None  # what Telegram says about the destination channel (when copies were hidden)
+    hidden_stop: bool = False  # the run stopped by itself: Telegram hides every copy it read back
     aborted: Optional[str] = None
     stopped: bool = False
     first_new: Optional[int] = None
@@ -762,6 +859,30 @@ def _rows(unit: list, finals: list, news: list) -> list:
     return rows
 
 
+CHECK_FIRST = 3  # the first copies of any run are read back ...
+CHECK_EVERY = 10  # ... then every tenth (all of them when Telegram restricts the source)
+STOP_IF_HIDDEN = 3  # /shift stops when Telegram hides this many new copies and not one of those read back is shown
+
+
+async def check_copies(client, dst_peer, unit: list, news: list, res: RepostResult) -> None:
+    """Read the new posts back: does TELEGRAM itself hide them (a restriction mark, or its notice as the text)? That
+    is not the bot copying a notice - Telegram restricts the new post as well, usually because of a file or a link in
+    it. What is found goes into `res` (checked, hidden, hidden_why). Never fails the copy: it is only a look."""
+    try:
+        got = await fetch_messages(client, dst_peer, [n.id for n in news])
+    except Exception as e:  # a look that fails is not a failed copy (the post exists and is registered already)
+        log.info("could not read the new posts back: %s", type(e).__name__)
+        return
+    for old, new in zip(unit, got):
+        if is_skippable(new):
+            continue
+        res.checked += 1
+        why = message_restriction(new)
+        if why:
+            res.hidden.append((old.id, _has_media([old])))
+            res.hidden_why = res.hidden_why or why
+
+
 @dataclass
 class CopyJob:
     """Everything the copy loop needs to know; /repost and /shift each fill it in their own way."""
@@ -779,6 +900,9 @@ class CopyJob:
     use_saved: bool = False  # a post Telegram holds back is rebuilt from its saved copy (otherwise it is left out)
     units: Optional[Callable[[], AsyncIterator[list]]] = None  # where the posts come from (default: read the channel)
     from_db: bool = False  # `units` are saved posts: they are all rebuilt, none is read from Telegram
+    restricted: Optional[str] = None  # what Telegram says about the source channel itself (a copyright strike), if anything
+    no_media: bool = False  # the copies get no files: text, formatting and buttons only (/shift --no-media)
+    stop_if_hidden: int = 0  # stop when this many copies were read back and Telegram hides every one (0: never)
 
 
 async def copy_loop(
@@ -795,7 +919,10 @@ async def copy_loop(
     src, dst = job.src_peer, job.dst_peer
     seen: set = set()  # old ids read in this run (a reply to one of them needs its copy)
     res, st, streak = RepostResult(used_saved=job.use_saved or job.from_db, from_db=job.from_db), _State(), 0
+    res.no_media = job.no_media
     units = job.units() if job.units is not None else iter_units(client, src, job.first_id, job.last_id)
+    if job.no_media:
+        units = single_posts(units)
     async for unit in units:
         if should_stop and should_stop():
             res.stopped = True
@@ -806,20 +933,43 @@ async def copy_loop(
             res.skipped_done += 1
             continue
         rows = await job.saved(ids) if (job.saved and not job.from_db) else {}
-        why = None if job.from_db else held_back_reason(unit, rows)
-        from_saved = job.from_db  # saved posts already
-        if why:  # Telegram shows a notice instead of this post: the notice must not become the copy
+        if job.from_db:
+            source, why = "saved", None
+        else:
+            source, why = copy_source(unit, rows, restricted=job.restricted, use_saved=job.use_saved)
+        if why:  # Telegram restricts this post (or the whole channel): what it hands out now is not trusted
             res.held_why = res.held_why or why
-            standing = stand_ins(unit, rows) if job.use_saved else None
+        from_saved = job.from_db  # saved posts already
+        standing = None
+        if source == "saved" and not job.from_db:
+            standing = stand_ins(unit, rows)
             if standing is None:
-                log.info("post %s is held back by Telegram (%s) and has no saved copy to take its place", ids[0], why)
-                res.held_back.append(ids[0])
-                continue
+                source = "skip"
+        if source == "skip":
+            log.info("post %s: Telegram shows only its notice (%s) and My posts has no saved copy to use", ids[0], why)
+            res.held_back.append(ids[0])
+            continue
+        if standing is not None:
             unit, from_saved = standing, True
+        no_forward, left_out = bool(why), 0
+        if job.no_media:  # the copy is the text with its formatting and buttons; its file stays where it is
+            unit, left_out = without_files(unit)
+            if not unit:  # nothing but a file: there is nothing to post without it, and nothing waits for it
+                res.file_only.append(ids[0])
+                seen.difference_update(ids)
+                continue
+            no_forward = no_forward or bool(left_out)  # a copy by Telegram would carry the file
         finals = [final_state(m, job.opts, job.links) for m in unit]
+        if any(is_placeholder_text(f.text) for f in finals):  # Telegram's notice is never posted, whatever it came from
+            log.info("post %s: its text is Telegram's notice - not copied", ids[0])
+            res.held_back.append(ids[0])
+            res.held_why = res.held_why or PLACEHOLDER_NOTE
+            continue
         try:
             reply_to = new_parent(unit[0], job.idmap, seen)
-            news = await copy_unit(client, src, unit, finals, st, reply_to, dst, from_saved=from_saved)
+            news = await copy_unit(
+                client, src, unit, finals, st, reply_to, dst, from_saved=from_saved, no_forward=no_forward
+            )
             await settle_buttons(client, dst, finals, news, st, pause=settle_pause)
         except Exception as e:  # one bad post must not stop the others
             name = type(e).__name__
@@ -845,13 +995,27 @@ async def copy_loop(
             job.idmap[old.id] = new.id
         if reply_parent(unit[0]) is not None and reply_to is None:
             res.reply_dropped += 1
+        if why and not from_saved:
+            res.live_restricted += 1
+        res.files_left_out += left_out
+        if why or res.copied_units < CHECK_FIRST or res.copied_units % CHECK_EVERY == 0:
+            await check_copies(client, dst, unit, news, res)
         res.copied_units += 1
         res.copied_messages += len(unit)
         res.first_new = res.first_new or news[0].id
         res.last_new = news[-1].id
         if progress:
             await progress(res)
+        if job.stop_if_hidden and res.checked >= job.stop_if_hidden and len(res.hidden) == res.checked:
+            log.info("Telegram hides every copy read back (%s): stopping", res.checked)
+            res.stopped = res.hidden_stop = True  # more copies would only be more posts that nobody can see
+            break
         await asyncio.sleep(delay)
+    if res.hidden:  # Telegram hides copies: is the destination itself restricted too?
+        try:
+            _top, res.dst_restricted, _err = await probe_channel(client, dst)
+        except Exception:  # only an extra for the report
+            log.debug("could not ask Telegram about the destination channel", exc_info=True)
     res.forwarded, res.rebuilt, res.repaired = st.forwarded, st.rebuilt, st.repaired
     res.replies, res.reply_lost = st.replies, st.reply_lost
     res.from_saved, res.media_lost = st.from_saved, st.media_lost

app/shift_engine.py

@@ -4,14 +4,22 @@
 It is the /repost copy loop pointed at two channels. Links from a shifted post to another post of the source channel
 follow the content: when that post was shifted too, the link points at its copy in the destination.
 
-Two optional extras are remembered with the shift (job marks), so that continuing a stopped shift keeps them:
-  -c    afterwards every link to a post of the source, in ANY connected channel, is pointed at the copy
-        (crosslinks.relink_after_shift; the source itself is only read);
-  -all  afterwards the destination gets the source's name, description and profile photo (profile_copy.copy_profile).
-
-A source that Telegram holds back (a copyright strike: it shows a notice instead of its posts) is copied from what
-My posts saved of the posts (repost_engine: held_back_reason / stand_ins). A source that can't be read at all (a banned
-or private channel) is copied from My posts alone: mark "fromdb", set when the plan was made that way.
+Three optional extras are remembered with the shift (job marks), so that continuing a stopped shift keeps them:
+  -c          afterwards every link to a post of the source, in ANY connected channel, is pointed at the copy
+              (crosslinks.relink_after_shift; the source itself is only read);
+  -all        afterwards the destination gets the source's name, description and profile photo
+              (profile_copy.copy_profile);
+  --no-media  the copies get no files: every post is copied as its text, with its formatting and buttons (mark "nomedia";
+              repost_engine.without_files). A post that is only a file has nothing to copy and is left out.
+
+When Telegram hides the first new copies (read back right after posting) the shift stops by itself - more copies would
+only be more posts nobody can see - and goes on only when the owner presses Continue (mark "hiddenok").
+
+A source that Telegram restricts (a copyright strike: "This channel is unavailable due to copyright infringement", its
+posts marked or shown as a notice) is copied from what My posts saved of its posts - every post that has a saved copy,
+however intact Telegram's version looks (repost_engine.copy_source). A post without a saved copy is copied as Telegram
+hands it out if that is the post itself, and skipped if it is only the notice. A source that can't be read at all (a
+banned or private channel) is copied from My posts alone: mark "fromdb", set when the plan was made that way.
 """
 from __future__ import annotations
 
@@ -22,6 +30,7 @@ from typing import Any, Awaitable, Callable, Optional
 from .crosslinks import CrossResult, relink_after_shift, shift_links
 from .profile_copy import copy_profile
 from .repost_engine import (
+    STOP_IF_HIDDEN,
     CopyJob,
     DeleteResult,
     RepostOptions,
@@ -34,6 +43,7 @@ from .repost_engine import (
     saved_lookup,
     saved_units,
 )
+from .restrictions import probe_channel
 from .tgutil import peer_of
 
 log = logging.getLogger(__name__)
@@ -46,12 +56,14 @@ def source_of(shift) -> SimpleNamespace:
     )
 
 
-async def plan_from_my_posts(db, src, first_id=None, last_id=None, unreadable=None) -> Optional[RepostPlan]:
+async def plan_from_my_posts(
+    db, src, first_id=None, last_id=None, unreadable=None, no_media: bool = False
+) -> Optional[RepostPlan]:
     """The plan of a shift made from My posts alone, for a source that can't be read; None when My posts has nothing
     of that channel (in that range)."""
     plan = plan_from_saved(
         src, await db.sent_posts_from(src.id, first_id or 1), RepostOptions(), first_id=first_id, last_id=last_id,
-        unreadable=unreadable,
+        unreadable=unreadable, no_media=no_media,
     )
     return None if plan.error else plan
 
@@ -73,7 +85,14 @@ async def run_shift(
     src = source_of(shift)
     idmap = dict(await db.shift_pairs(shift.id, alive_only=False))
     links = shift_links(src, dst, idmap)
-    from_db = "fromdb" in await db.marks_of(shift.id)  # the source can't be read: its posts come from My posts alone
+    marks = await db.marks_of(shift.id)
+    from_db = "fromdb" in marks  # the source can't be read: its posts come from My posts alone
+    restricted = None
+    if not from_db:  # does Telegram restrict the source channel itself (a copyright strike)? Then My posts is used
+        try:
+            _top, restricted, _err = await probe_channel(client, peer_of(src))
+        except Exception:  # unknown is treated as "not restricted": marks on single posts and the read-back still guard
+            log.warning("could not ask Telegram whether the source channel is restricted", exc_info=True)
     job = CopyJob(
         src_peer=peer_of(src),
         dst_peer=peer_of(dst),
@@ -88,6 +107,9 @@ async def run_shift(
         use_saved=True,
         units=(lambda: saved_units(db, src.id, shift.first_id, shift.last_id)) if from_db else None,
         from_db=from_db,
+        restricted=restricted,
+        no_media="nomedia" in marks,
+        stop_if_hidden=0 if "hiddenok" in marks else STOP_IF_HIDDEN,  # the owner said to go on although Telegram hides them
     )
     res = await copy_loop(client, job, delay=delay, progress=progress, should_stop=should_stop, settle_pause=settle_pause)
     if not res.stopped and not res.aborted:

tests/fakes.py

@@ -139,6 +139,7 @@ class FakeChannelClient:
         self.users = {}  # user id -> types.User, for participant lists
         self.cant_delete_at_all = set()  # ids nobody can delete (userbot included)
         self.restriction = None  # text: Telegram restricts the whole channel (a copyright strike)
+        self.restrict_new = None  # f(new message) -> text or None: Telegram itself restricts a NEW post (a blocked file ...)
         self.about = ""  # the channel's description
         self.photo_id = None  # the channel's profile photo (its bytes are in _photo_data)
         self._photo_data = {}  # photo id -> bytes; shared by all channels next to each other
@@ -236,6 +237,10 @@ class FakeChannelClient:
             hide = True
             self.hide_keyboard_on_create -= 1
         self._set_markup(m, markup, hide=hide)
+        if self.restrict_new is not None:
+            why = self.restrict_new(m)
+            if why:
+                m.restriction_reason = [types.RestrictionReason(platform="all", reason="copyright", text=why)]
         self.msgs[m.id] = m
         self._remember_media(m)
         return m

tests/test_held_back.py

@@ -6,6 +6,7 @@ import asyncio
 from telethon import functions, types
 
 from app.repost_engine import RepostOptions, plan_repost, saved_lookup
+from app.restrictions import PLACEHOLDER_NOTE
 from app.shift_engine import run_shift
 from app.tgutil import button_url
 
@@ -13,7 +14,7 @@ from .dbutil import db_url_for, fresh_db
 from .fakes import FakeChannelClient, make_msg, markup, photo_media, rpc, service_msg, url_btn
 from .harness import norm
 from .test_repost_flow import fill, make_app
-from .test_restricted import NOTICE, strike
+from .test_restricted import CHANNEL_NOTICE, NOTICE, strike
 from .test_shift import SRC, old_posts, shift_app
 from .test_sync import saved
 
@@ -135,24 +136,30 @@ def test_a_post_that_telegram_shows_as_it_is_still_comes_from_the_channel(tmp_pa
     run(go())
 
 
-def test_a_post_with_telegrams_mark_that_is_still_handed_out_as_it_is_is_an_ordinary_post(tmp_path):
+def test_a_post_with_telegrams_mark_comes_from_my_posts_even_when_telegram_still_hands_it_out(tmp_path):
+    """The rule of a struck channel: what My posts saved wins over what Telegram hands out now - however intact that
+    looks. Only a post without a saved copy is copied as Telegram hands it out (it is the post itself, not a notice)."""
+
     async def go():
         orig = originals()
         live = dict(orig)
         live[2] = strike(make_msg(2, "Episode 2 is out", [types.MessageEntityBold(0, 9)], markup=orig[2].reply_markup))
-        live[3] = strike(make_msg(3, "Caption with link", [types.MessageEntityTextUrl(13, 4, "https://t.me/srcch/5")], media=photo_media(31)))
+        live[3] = strike(make_msg(3, "Caption as Telegram hands it out", media=photo_media(31)))  # differs from My posts
         src = FakeChannelClient(live, username="srcch", title="Anime Source")
         dst = src.add_channel(FakeChannelClient(old_posts(), username="dstch", cid=2, title="Anime Backup"))
+        for m in orig.values():
+            src._remember_media(m)
         db = await fresh_db(db_url_for(tmp_path))
         await db.save_channel(1, 5, "Anime Source", "srcch", 1)
         await db.save_channel(2, 6, "Anime Backup", "dstch", 1)
         await save_rows(db, orig, skip=(2,))  # no saved copy of post 2; post 3 has one
         await db.create_shift("s1", src=SRC, dst_channel_id=2, first_id=1, last_id=6, user_id=1)
         res = await run_shift(src, db, await db.get_shift("s1"), await db.get_channel(2), 99, delay=0, settle_pause=0)
-        # the live posts are what they are: nothing is rebuilt from My posts, nothing is skipped
-        assert res.from_saved == 0 and res.held_back == [] and res.failed == [] and res.copied_units == 5
+        assert res.held_back == [] and res.failed == [] and res.copied_units == 5
+        assert res.from_saved == 1  # post 3
+        assert res.live_restricted == 1  # post 2: marked, but nothing saved to use instead and its content is intact
         assert shown(dst) == {11: "Welcome", 12: "Episode 2 is out", 13: "Caption with link", 14: "Fifth", 15: "Sixth"}
-        assert 3 in forwarded_ids(dst)
+        assert forwarded_ids(dst) == [1, 6]  # a restricted post is never copied by Telegram itself
         await db.close()
 
     run(go())
@@ -178,14 +185,54 @@ def test_a_post_that_telegram_empties_out_is_held_back_too(tmp_path):
     run(go())
 
 
-def test_a_text_that_my_posts_has_too_is_the_owners_own_wording_not_a_notice(tmp_path):
+def test_telegrams_notice_is_never_posted_even_when_my_posts_holds_it_as_the_text(tmp_path):
+    """A saved copy made from a restricted post can hold Telegram's notice as its text (an older version did that), and a
+    post may really say it: either way it is indistinguishable from the notice, so it is not copied - and reported."""
+
     async def go():
         orig = originals()
-        orig[6] = make_msg(6, NOTICE)  # the owner really wrote this
+        orig[6] = make_msg(6, NOTICE)
         src, dst, db, dst_row, shift = await held_setup(tmp_path, held=(2, 3, 5), orig=orig, flagged=False)
         res = await run_shift(src, db, shift, dst_row, 99, delay=0, settle_pause=0)
-        assert res.from_saved == 3 and res.held_back == [] and res.failed == []
-        assert dst.msgs[15].message == NOTICE and 6 in forwarded_ids(dst)  # copied as it is
+        assert res.from_saved == 3 and res.held_back == [6] and res.failed == []
+        assert shown(dst) == {11: "Welcome", 12: "Episode 2 is out", 13: "Caption with link", 14: "Fifth"}
+        assert not any(NOTICE in (m.message or "") for m in dst.msgs.values())
+        assert 6 not in forwarded_ids(dst) and res.held_why == PLACEHOLDER_NOTE
+        await db.close()
+
+    run(go())
+
+
+def test_a_saved_copy_that_holds_the_notice_is_not_used_for_a_restricted_post(tmp_path):
+    async def go():
+        orig = originals()
+        orig[5] = make_msg(5, "Fifth")  # answers nothing
+        src, dst, db, dst_row, shift = await held_setup(tmp_path, held=(2, 3), no_row=(2,), orig=orig)
+        await db.create_post(  # what an older version saved when it synced the struck post
+            channel_id=1, message_id=2, status="sent", source="adopted", created_by=1, link_preview=False,
+            text=NOTICE, entities=[], buttons=[], media_kind=None, media_file_id=None,
+        )
+        res = await run_shift(src, db, shift, dst_row, 99, delay=0, settle_pause=0)
+        # post 2: Telegram shows the notice and the saved copy is the notice too: nothing to copy; post 3 is fine
+        assert res.held_back == [2] and res.from_saved == 1 and res.failed == []
+        assert shown(dst) == {11: "Welcome", 12: "Caption with link", 13: "Fifth", 14: "Sixth"}
+        assert not any(NOTICE in (m.message or "") for m in dst.msgs.values())
+        await db.close()
+
+    run(go())
+
+
+def test_an_answer_to_a_post_that_could_not_be_copied_follows_when_that_post_has_its_copy(tmp_path):
+    async def go():
+        src, dst, db, dst_row, shift = await held_setup(tmp_path, held=(2,), no_row=(2,))  # post 5 answers post 2
+        res = await run_shift(src, db, shift, dst_row, 99, delay=0, settle_pause=0)
+        assert res.held_back == [2] and res.failed == [(5, "ReplyParentMissing")]  # it would answer nothing
+        assert shown(dst) == {11: "Welcome", 12: "Caption with link", 13: "Sixth"}
+        await save_rows(db, {2: originals()[2]})  # the saved copy arrives
+        again = await run_shift(src, db, shift, dst_row, 99, delay=0, settle_pause=0)
+        assert again.held_back == [] and again.failed == [] and again.copied_units == 2
+        assert dst.msgs[14].message == "Episode 2 is out" and dst.msgs[15].message == "Fifth"
+        assert dst.msgs[15].reply_to.reply_to_msg_id == 14  # the answer answers the copy
         await db.close()
 
     run(go())
@@ -376,22 +423,26 @@ def buttons(app):
 def test_the_shift_plan_and_result_explain_what_comes_from_my_posts(tmp_path):
     async def go():
         app, dst = await held_app(tmp_path, no_row=(5,))
-        app.tg.restriction = NOTICE
+        app.tg.restriction = CHANNEL_NOTICE  # the whole channel is struck: My posts is used for every post it has
         await app.text("/shift @srcch @dstch")
         plan = norm(app.out.last_text)
-        assert "Telegram restricts Anime Source itself" in plan
-        assert "Telegram holds back 2 post(s) of Anime Source" in plan and "copied from what My posts saved of them" in plan
-        assert "holds back 1 more post(s)" in plan and "no saved copy" in plan and "/import" in plan
+        assert "Telegram restricts Anime Source itself" in plan and "unavailable due to copyright infringement" in plan
+        assert "4 post(s) of Anime Source are restricted by Telegram" in plan
+        assert "copied from what My posts saved of them" in plan and "not from what Telegram shows now" in plan
+        assert "Telegram shows only its notice for 1 more post(s)" in plan and "no saved copy" in plan and "/import" in plan
         assert "4 posts" in plan  # 1, 2, 3 and 6 are copied
+        assert "The new posts are read back" in plan
         assert not any(isinstance(r, functions.messages.SendMessageRequest) for r in dst.requests)  # a plan posts nothing
 
         await app.press(app.out.callback_data("Start copying"))
         done = norm(app.out.last_text)
-        assert "2 post(s) are held back by Telegram" in done and "copied from what My posts saved of them" in done
-        assert "1 post(s) are held back by Telegram and My posts has no saved copy of them" in done and "#5" in done
+        assert "4 post(s) are restricted by Telegram" in done and "copied from what My posts saved of them" in done
+        assert "1 post(s): Telegram shows only its notice and My posts has no saved copy of them" in done and "#5" in done
         assert "not everything was copied" in done  # so the rest can be tried again later
         assert any("Try the rest again" in t for t in buttons(app))
+        assert "hides" not in done  # nothing the new posts show is restricted
         assert not any(NOTICE in (m.message or "") for m in dst.msgs.values())
+        assert forwarded_ids(dst) == []  # a struck channel: nothing is copied by Telegram itself
 
         # the missing copy arrives (a backup put back with /import): the rest can be copied
         await save_rows(app.db, {5: originals()[5]})

tests/test_no_media.py

@@ -0,0 +1,279 @@
+"""/shift --no-media: no file is copied - every post goes out as its text with its formatting and buttons. For a source
+whose files Telegram blocks (a copyright strike), where a post that carries the file is hidden wherever it is posted."""
+import asyncio
+
+from telethon import functions, types
+
+from app.handlers.shift import USAGE, parse_shift_args
+from app.repost_engine import RepostOptions, plan_repost, saved_lookup, without_files
+from app.shift_engine import plan_from_my_posts, run_shift
+from app.tgutil import button_url
+
+from .fakes import make_msg, photo_media
+from .harness import norm
+from .test_held_back import buttons, held_app, held_setup, forwarded_ids, originals, save_rows, shown, unreadable
+from .test_restricted import CHANNEL_NOTICE
+from .test_shift import SRC
+
+
+def run(coro):
+    return asyncio.run(coro)
+
+
+def with_files():
+    """The source: text posts, a photo with a caption, a post that is only a photo, an album and an answer to the photo."""
+    orig = originals()  # 1 text, 2 text + button, 3 photo + caption, 4 service, 5 answers 2, 6 text
+    orig[7] = make_msg(7, "", media=photo_media(71))  # only a photo
+    orig[8] = make_msg(8, "", media=photo_media(81), grouped_id=80)  # an album: no caption on the first picture ...
+    orig[9] = make_msg(9, "album caption", [types.MessageEntityBold(0, 5)], media=photo_media(82), grouped_id=80)
+    orig[10] = make_msg(10, "Reply to the photo", reply_to=7)
+    return orig
+
+
+# ============================================================================================== the option
+def test_the_option_is_understood_in_every_spelling():
+    for raw in (
+        "@a @b --no-media",
+        "@a @b -no-media",
+        "@a @b --nomedia",
+        "@a @b -nomedia",
+        "@a @b --NO-MEDIA",
+        "@a @b --no_media",
+        "@a @b —no-media",  # a phone keyboard turns -- into a long dash
+        "@a @b –no-media",
+        "@a @b -- no-media",
+        "--no-media @a @b",
+    ):
+        a, b, first, last, flags = parse_shift_args(raw)
+        assert (a, b, first, last) == ("@a", "@b", None, None) and flags.no_media and not flags.links and not flags.profile, raw
+    a, b, first, last, flags = parse_shift_args("@a @b 5 9 -c --no-media -all")
+    assert (a, b, first, last) == ("@a", "@b", 5, 9) and flags.links and flags.profile and flags.no_media
+    # a channel id is not an option, and nothing is switched on by accident
+    a, b, first, last, flags = parse_shift_args("-1001234567890 @b")
+    assert (a, b) == ("-1001234567890", "@b") and not flags.no_media
+    try:
+        parse_shift_args("@a @b --nomediaa")
+    except ValueError as e:
+        assert "--no-media" in str(e) and "-all" in str(e)
+    else:
+        raise AssertionError("an unknown option must be refused")
+
+
+def test_the_usage_describes_the_option():
+    assert "--no-media" in USAGE and "no file is copied" in USAGE
+
+
+def test_without_files_leaves_a_text_post_and_drops_a_post_that_was_only_a_file():
+    photo_caption = make_msg(3, "Caption", media=photo_media(31))
+    only_photo = make_msg(7, "", media=photo_media(71))
+    plain = make_msg(1, "Welcome")
+    kept, lost = without_files([plain, photo_caption, only_photo])
+    assert [m.id for m in kept] == [1, 3] and lost == 2
+    assert kept[1].media is None and photo_caption.media is not None  # the original message is not changed
+    assert without_files([only_photo]) == ([], 1)
+
+
+# ======================================================================================== the copy loop
+def test_the_files_stay_out_and_everything_else_is_copied(tmp_path):
+    async def go():
+        src, dst, db, dst_row, shift = await held_setup(tmp_path, held=(), orig=with_files(), last=10)
+        await db.set_mark("s1", "nomedia")
+        res = await run_shift(src, db, shift, dst_row, 99, delay=0, settle_pause=0)
+        assert (res.failed, res.aborted, res.held_back) == ([], None, []) and res.no_media
+        assert res.copied_units == 7 and res.files_left_out == 2 and res.file_only == [7, 8]
+        # copies: 1->11 2->12 3->13 5->14 6->15 9->16 10->17 (4 is a service message, 7 and 8 are only files)
+        assert shown(dst) == {
+            11: "Welcome", 12: "Episode 2 is out", 13: "Caption with link", 14: "Fifth", 15: "Sixth",
+            16: "album caption", 17: "Reply to the photo",
+        }
+        assert all(m.media is None for i, m in dst.msgs.items() if i > 10)  # no file anywhere
+        assert not sent_media(dst)
+        # a post without a file is still copied by Telegram itself - the answer to a post that is only a file too: it has
+        # no reply to keep any more
+        assert forwarded_ids(dst) == [1, 6, 10]
+        # text, formatting, buttons and the links between the copies are kept
+        assert [(e.offset, e.length) for e in dst.msgs[16].entities] == [(0, 5)]
+        btn = dst.msgs[12].reply_markup.rows[0].buttons[0]
+        assert (btn.text, button_url(btn)) == ("Download", "https://t.me/goku?start=AAA")
+        assert dst.msgs[13].entities[0].url == "https://t.me/dstch/14"
+        assert dst.msgs[14].reply_to.reply_to_msg_id == 12
+        # the answer to a post that is only a file answers nothing - and does not wait for a copy that never comes
+        assert dst.msgs[17].reply_to is None and res.reply_dropped == 1
+        # My posts knows the copies as they are: text, no file
+        row = await db.find_by_message(2, 13)
+        assert (row.text, row.media_kind, row.media_file_id) == ("Caption with link", None, None)
+        # the source was only read
+        assert not [r for r in src.requests if type(r).__name__.startswith(("Edit", "Delete", "Send", "Forward"))]
+        await db.close()
+
+    run(go())
+
+
+def sent_media(chan):
+    return [r for r in chan.requests if isinstance(r, (functions.messages.SendMediaRequest, functions.messages.SendMultiMediaRequest))]
+
+
+def test_without_the_option_the_files_are_copied_as_before(tmp_path):
+    async def go():
+        src, dst, db, dst_row, shift = await held_setup(tmp_path, held=(), orig=with_files(), last=10)
+        res = await run_shift(src, db, shift, dst_row, 99, delay=0, settle_pause=0)
+        assert not res.no_media and res.files_left_out == 0 and res.file_only == []
+        assert dst.msgs[13].media.photo.id == 31
+        await db.close()
+
+    run(go())
+
+
+def test_a_struck_channel_is_copied_from_my_posts_without_files(tmp_path):
+    async def go():
+        src, dst, db, dst_row, shift = await held_setup(tmp_path, held=(1, 2, 3, 5, 6, 7), orig=with_files(), last=7)
+        src.restriction = CHANNEL_NOTICE
+        await db.set_mark("s1", "nomedia")
+        res = await run_shift(src, db, shift, dst_row, 99, delay=0, settle_pause=0)
+        assert (res.failed, res.aborted, res.held_back) == ([], None, [])
+        assert res.from_saved == 5 and res.files_left_out == 1 and res.file_only == [7]
+        assert shown(dst) == {11: "Welcome", 12: "Episode 2 is out", 13: "Caption with link", 14: "Fifth", 15: "Sixth"}
+        assert all(m.media is None for i, m in dst.msgs.items() if i > 10) and not sent_media(dst) and not forwarded_ids(dst)
+        await db.close()
+
+    run(go())
+
+
+def test_a_source_that_cannot_be_read_is_copied_from_my_posts_without_files(tmp_path):
+    async def go():
+        src, dst, db, dst_row, shift = await held_setup(tmp_path, held=(), orig=with_files(), last=10)
+        unreadable(src)
+        await db.set_mark("s1", "fromdb")
+        await db.set_mark("s1", "nomedia")
+        res = await run_shift(src, db, shift, dst_row, 99, delay=0, settle_pause=0)
+        assert res.from_db and res.failed == [] and res.aborted is None
+        # My posts has no album: its two pictures are two saved posts - the one with the caption goes out as text
+        assert res.files_left_out == 2 and res.file_only == [7, 8]  # 3 and 9 lost their file; 7 and 8 are only files
+        assert shown(dst) == {
+            11: "Welcome", 12: "Episode 2 is out", 13: "Caption with link", 14: "Fifth", 15: "Sixth",
+            16: "album caption", 17: "Reply to the photo",
+        }
+        assert not sent_media(dst)
+        await db.close()
+
+    run(go())
+
+
+def test_a_media_only_post_is_skipped_but_never_makes_the_shift_incomplete(tmp_path):
+    async def go():
+        orig = {1: make_msg(1, "Welcome"), 2: make_msg(2, "", media=photo_media(21)), 3: make_msg(3, "Third")}
+        src, dst, db, dst_row, shift = await held_setup(tmp_path, held=(), orig=orig, last=3)
+        await db.set_mark("s1", "nomedia")
+        res = await run_shift(src, db, shift, dst_row, 99, delay=0, settle_pause=0)
+        assert res.file_only == [2] and res.held_back == [] and res.failed == [] and res.copied_units == 2
+        await db.close()
+
+    run(go())
+
+
+# ================================================================================================ the plan
+def test_the_plan_counts_posts_without_their_files(tmp_path):
+    async def go():
+        src, dst, db, dst_row, shift = await held_setup(tmp_path, held=(), orig=with_files(), last=10)
+        lookup = saved_lookup(db, 1)
+        plan = await plan_repost(src, SRC, RepostOptions(), saved=lookup, use_saved=True, no_media=True)
+        # 1 2 3 5 6 9 10 are copied (3 and 9 without their file); 7 and 8 are only files; 4 is a service message
+        assert (plan.units, plan.files_left_out, plan.file_only, plan.service) == (7, 2, 2, 1)
+        assert (plan.media, plan.albums, plan.messages, plan.no_media) == (0, 0, 7, True)
+        full = await plan_repost(src, SRC, RepostOptions(), saved=lookup, use_saved=True)
+        assert (full.units, full.media, full.albums, full.files_left_out, full.file_only) == (8, 4, 1, 0, 0)
+        await db.close()
+
+    run(go())
+
+
+def test_the_plan_from_my_posts_counts_posts_without_their_files(tmp_path):
+    async def go():
+        src, dst, db, dst_row, shift = await held_setup(tmp_path, held=(), orig=with_files(), last=10)
+        ch = await db.get_channel(1)
+        plan = await plan_from_my_posts(db, ch, no_media=True)
+        assert (plan.units, plan.files_left_out, plan.file_only, plan.media) == (7, 2, 2, 0)
+        await db.close()
+
+    run(go())
+
+
+def test_a_range_of_nothing_but_files_is_an_error_not_an_empty_plan(tmp_path):
+    async def go():
+        orig = {1: make_msg(1, "Welcome"), 2: make_msg(2, "", media=photo_media(21)), 3: make_msg(3, "", media=photo_media(31))}
+        src, dst, db, dst_row, shift = await held_setup(tmp_path, held=(), orig=orig, last=3)
+        plan = await plan_repost(src, SRC, RepostOptions(), first_id=2, last_id=3, saved=saved_lookup(db, 1), use_saved=True, no_media=True)
+        assert plan.units == 0 and plan.file_only == 2 and "only a file" in plan.error
+        await db.close()
+
+    run(go())
+
+
+# ========================================================================================= the owner's view
+def test_the_command_plans_runs_and_reports_a_shift_without_files(tmp_path):
+    async def go():
+        app, dst = await held_app(tmp_path, held=())
+        await app.text("/shift @srcch @dstch --no-media")
+        plan = norm(app.out.last_text)
+        assert "--no-media" in plan and "no file is copied" in plan
+        assert "1 post(s) carry a file that stays out" in plan and "5 posts" in plan
+        assert not any(isinstance(r, functions.messages.SendMessageRequest) for r in dst.requests)  # a plan posts nothing
+
+        await app.press(app.out.callback_data("Start copying"))
+        done = norm(app.out.last_text)
+        assert "5 message(s) copied" in done and "(without files)" in done
+        assert "1 post(s) were copied as text only" in done and "their file was left out" in done
+        assert all(m.media is None for i, m in dst.msgs.items() if i > 10)
+        sid = app.out.callback_data("Undo").split(":")[1]
+        assert await app.db.marks_of(sid) == {"nomedia"}
+        await app.db.close()
+
+    run(go())
+
+
+def test_a_shift_that_is_continued_goes_on_without_files(tmp_path):
+    async def go():
+        app, dst = await held_app(tmp_path, no_row=(5,))
+        app.tg.restriction = CHANNEL_NOTICE
+        await app.text("/shift @srcch @dstch --no-media")
+        await app.press(app.out.callback_data("Start copying"))
+        assert any("Try the rest again" in t for t in buttons(app))  # post 5: only the notice, nothing saved
+        await save_rows(app.db, {5: make_msg(5, "Fifth", reply_to=2, media=photo_media(51))})  # the saved copy has a photo
+        await app.press(app.out.callback_data("Try the rest again"))
+        assert dst.msgs[15].message == "Fifth" and dst.msgs[15].media is None  # the mark kept the option
+        assert not sent_media(dst)
+        await app.db.close()
+
+    run(go())
+
+
+def test_the_whole_story_telegram_hides_posts_with_files_and_without_them_it_does_not(tmp_path):
+    """The case behind the option: the new copies are hidden by Telegram because of their files."""
+
+    async def go():
+        for no_media in (False, True):
+            app, dst = await held_app(tmp_path / str(no_media), held=())
+            app.tg.restriction = CHANNEL_NOTICE
+            dst.restrict_new = lambda m: CHANNEL_NOTICE if m.media is not None else None  # the photo is blocked
+            await app.text("/shift @srcch @dstch" + (" --no-media" if no_media else ""))
+            await app.press(app.out.callback_data("Start copying"))
+            done = norm(app.out.last_text)
+            if no_media:
+                assert "hides" not in done and "(without files)" in done
+            else:
+                assert "Telegram itself hides 1 of the 5 new post(s) I read back" in done and "--no-media" in done
+            await app.db.close()
+
+    run(go())
+
+
+def test_the_repost_command_does_not_take_the_option(tmp_path):
+    async def go():
+        from .test_repost_flow import make_app
+
+        app = await make_app(tmp_path, msgs={1: make_msg(1, "First post")})
+        await app.text("/repost --no-media")
+        assert "Unknown" in norm(app.out.last_text) or "unknown" in norm(app.out.last_text).lower()
+        await app.db.close()
+
+    run(go())

tests/test_restricted.py

@@ -10,6 +10,7 @@ from .test_sync import foreign, looked, old_msg, opts, run, world
 from .test_syncer import dms, make_app, three as three_posts
 
 NOTICE = "This message couldn't be displayed on your device due to copyright infringement."
+CHANNEL_NOTICE = "This channel is unavailable due to copyright infringement."  # what Telegram says of a struck channel
 
 
 def strike(m, text=NOTICE):

tests/test_restricted_source.py

@@ -0,0 +1,418 @@
+"""A source channel that TELEGRAM restricts (a copyright strike: "This channel is unavailable due to copyright
+infringement"): /shift copies it from what My posts saved - every post that has a saved copy, whatever Telegram hands out
+of it now - and never copies Telegram's notice. Every copy is read back, so the report can say when Telegram itself hides
+the NEW posts (a blocked file, a restricted destination) instead of the bot being blamed for copying a notice."""
+import asyncio
+
+from telethon import functions
+
+from app.repost_engine import CHECK_FIRST, RepostOptions, plan_repost, saved_lookup
+from app.restrictions import message_restriction
+from app.shift_engine import run_shift
+from app.tgutil import button_url
+
+from .fakes import make_msg, rpc
+from .harness import norm
+from .test_held_back import (
+    buttons,
+    held_app,
+    held_setup,
+    held_version,
+    forwarded_ids,
+    originals,
+    shown,
+    unreadable,
+)
+from .test_restricted import CHANNEL_NOTICE, NOTICE
+from .test_shift import SRC
+
+
+def run(coro):
+    return asyncio.run(coro)
+
+
+COPIES = {11: "Welcome", 12: "Episode 2 is out", 13: "Caption with link", 14: "Fifth", 15: "Sixth"}  # as My posts has them
+
+
+def differ(src, ids):
+    """What Telegram hands out of these posts now is not what My posts saved (so the source of a copy can be told)."""
+    for i in ids:
+        old = src.msgs[i]
+        src.msgs[i] = make_msg(i, f"LIVE {i}", reply_to=old.reply_to.reply_to_msg_id if old.reply_to else None)
+
+
+async def struck(tmp_path, *, no_row=(), live_differs=True, **kw):
+    """The source is restricted as a whole; its posts are handed out intact (and, by default, differently from what My
+    posts saved) and carry no mark of their own - the case that went wrong."""
+    src, dst, db, dst_row, shift = await held_setup(tmp_path, held=kw.pop("held", ()), no_row=no_row, **kw)
+    src.restriction = CHANNEL_NOTICE
+    if live_differs:
+        differ(src, [i for i in (1, 2, 3, 5, 6) if i in src.msgs and not message_restriction(src.msgs[i])])
+    return src, dst, db, dst_row, shift
+
+
+# ======================================================================================== the copy loop
+def test_a_struck_channel_is_copied_from_my_posts_even_when_telegram_still_hands_the_posts_out(tmp_path):
+    async def go():
+        src, dst, db, dst_row, shift = await struck(tmp_path)
+        dst.fail["ForwardMessagesRequest"] = rpc("MessageIdInvalidError")  # what the real run met: nothing can be forwarded
+        res = await run_shift(src, db, shift, dst_row, 99, delay=0, settle_pause=0)
+        assert (res.failed, res.aborted, res.held_back) == ([], None, [])
+        assert res.from_saved == 5 and res.copied_units == 5 and res.live_restricted == 0 and res.held_why == CHANNEL_NOTICE
+        assert shown(dst) == COPIES  # My posts' texts - not "LIVE n"
+        assert forwarded_ids(dst) == []  # Telegram is not even asked to copy anything out of the struck channel
+        # text, formatting, buttons, media and links come from My posts
+        assert [(e.offset, e.length) for e in dst.msgs[12].entities] == [(0, 9)]
+        btn = dst.msgs[12].reply_markup.rows[0].buttons[0]
+        assert (btn.text, button_url(btn)) == ("Download", "https://t.me/goku?start=AAA")
+        assert dst.msgs[13].media.photo.id == 31
+        assert dst.msgs[13].entities[0].url == "https://t.me/dstch/14"  # the link to post 5 follows its copy
+        assert dst.msgs[14].reply_to.reply_to_msg_id == 12  # an answer answers the copy
+        # My posts knows the copies as they are, and the source was only read
+        assert (await db.find_by_message(2, 12)).text == "Episode 2 is out"
+        assert not [r for r in src.requests if type(r).__name__.startswith(("Edit", "Delete", "Send", "Forward"))]
+        await db.close()
+
+    run(go())
+
+
+def test_a_struck_channel_is_copied_from_my_posts_when_telegram_hands_out_only_the_notice(tmp_path):
+    """The same channel when the apps show the notice: the posts carry Telegram's mark, or only its words as the text."""
+
+    async def go():
+        for flagged in (True, False):
+            src, dst, db, dst_row, shift = await struck(tmp_path / str(flagged), held=(1, 2, 3, 5, 6), flagged=flagged)
+            res = await run_shift(src, db, shift, dst_row, 99, delay=0, settle_pause=0)
+            assert (res.failed, res.aborted, res.held_back) == ([], None, [])
+            assert res.from_saved == 5 and shown(dst) == COPIES
+            assert not any(NOTICE in (m.message or "") for m in dst.msgs.values())
+            await db.close()
+
+    run(go())
+
+
+def test_a_post_of_a_struck_channel_without_a_saved_copy_is_copied_as_telegram_hands_it_out(tmp_path):
+    async def go():
+        src, dst, db, dst_row, shift = await struck(tmp_path, no_row=(3, 5))
+        res = await run_shift(src, db, shift, dst_row, 99, delay=0, settle_pause=0)
+        assert (res.failed, res.aborted, res.held_back) == ([], None, [])
+        assert res.from_saved == 3 and res.live_restricted == 2  # 1 2 6 from My posts; 3 and 5 as they are now
+        assert shown(dst) == {11: "Welcome", 12: "Episode 2 is out", 13: "LIVE 3", 14: "LIVE 5", 15: "Sixth"}
+        assert forwarded_ids(dst) == []  # not even those: a restricted post is never copied by Telegram itself
+        await db.close()
+
+    run(go())
+
+
+def test_a_post_of_a_struck_channel_with_only_the_notice_and_no_saved_copy_is_skipped(tmp_path):
+    async def go():
+        src, dst, db, dst_row, shift = await struck(tmp_path, held=(3,), no_row=(3,), flagged=False)
+        res = await run_shift(src, db, shift, dst_row, 99, delay=0, settle_pause=0)
+        assert res.held_back == [3] and res.from_saved == 4 and res.failed == []
+        assert shown(dst) == {11: "Welcome", 12: "Episode 2 is out", 13: "Fifth", 14: "Sixth"}
+        assert not any(NOTICE in (m.message or "") for m in dst.msgs.values())
+        await db.close()
+
+    run(go())
+
+
+def test_a_saved_copy_that_is_telegrams_notice_is_never_posted(tmp_path):
+    async def go():
+        src, dst, db, dst_row, shift = await struck(tmp_path, no_row=(2, 3))
+        for mid in (2, 3):  # what an older version saved when it synced the struck channel
+            await db.create_post(
+                channel_id=1, message_id=mid, status="sent", source="adopted", created_by=1, link_preview=False,
+                text=NOTICE, entities=[], buttons=[], media_kind=None, media_file_id=None,
+            )
+        src.msgs[3] = held_version(originals()[3], flagged=False)  # post 3: the notice from Telegram as well
+        res = await run_shift(src, db, shift, dst_row, 99, delay=0, settle_pause=0)
+        # post 2: the saved copy is useless but Telegram hands the post out; post 3: nothing but notices
+        assert res.held_back == [3] and res.live_restricted == 1 and res.from_saved == 3 and res.failed == []
+        assert shown(dst) == {11: "Welcome", 12: "LIVE 2", 13: "Fifth", 14: "Sixth"}
+        assert not any(NOTICE in (m.message or "") for m in dst.msgs.values())
+        await db.close()
+
+    run(go())
+
+
+def test_a_source_that_cannot_be_read_never_gets_a_saved_notice_posted(tmp_path):
+    async def go():
+        src, dst, db, dst_row, shift = await held_setup(tmp_path, held=(), no_row=(2,))
+        await db.create_post(
+            channel_id=1, message_id=2, status="sent", source="adopted", created_by=1, link_preview=False,
+            text=NOTICE, entities=[], buttons=[], media_kind=None, media_file_id=None,
+        )
+        unreadable(src)
+        await db.set_mark("s1", "fromdb")
+        res = await run_shift(src, db, shift, dst_row, 99, delay=0, settle_pause=0)
+        assert res.copied_units == 4 and res.failed == []
+        assert shown(dst) == {11: "Welcome", 12: "Caption with link", 13: "Fifth", 14: "Sixth"}
+        await db.close()
+
+    run(go())
+
+
+def test_a_channel_nothing_restricts_is_copied_as_before(tmp_path):
+    async def go():
+        src, dst, db, dst_row, shift = await held_setup(tmp_path, held=())
+        differ(src, (1, 2, 3, 5, 6))
+        res = await run_shift(src, db, shift, dst_row, 99, delay=0, settle_pause=0)
+        assert res.from_saved == 0 and res.live_restricted == 0 and res.held_why is None and res.held_back == []
+        assert shown(dst) == {11: "LIVE 1", 12: "LIVE 2", 13: "LIVE 3", 14: "LIVE 5", 15: "LIVE 6"}  # what the channel says
+        await db.close()
+
+    run(go())
+
+
+def test_lifting_the_strike_makes_the_next_run_an_ordinary_one(tmp_path):
+    async def go():
+        src, dst, db, dst_row, shift = await struck(tmp_path, last=3)
+        stopper = {"n": 0}
+
+        async def prog(r):
+            stopper["n"] = r.copied_units
+
+        res = await run_shift(src, db, shift, dst_row, 99, delay=0, settle_pause=0, progress=prog, should_stop=lambda: stopper["n"] >= 1)
+        assert res.stopped and shown(dst) == {11: "Welcome"}  # post 1, from My posts
+        src.restriction = None  # the strike is lifted: the channel says what it says now
+        again = await run_shift(src, db, shift, dst_row, 99, delay=0, settle_pause=0)
+        assert again.held_why is None and again.from_saved == 0
+        assert shown(dst) == {11: "Welcome", 12: "LIVE 2", 13: "LIVE 3"}
+        await db.close()
+
+    run(go())
+
+
+# ================================================================================================ the plan
+def test_the_plan_counts_a_struck_channel_as_copied_from_my_posts(tmp_path):
+    async def go():
+        src, dst, db, dst_row, shift = await struck(tmp_path, no_row=(5,))
+        lookup = saved_lookup(db, 1)
+        plan = await plan_repost(src, SRC, RepostOptions(), saved=lookup, use_saved=True)
+        assert plan.channel_restricted == CHANNEL_NOTICE and plan.held_why == CHANNEL_NOTICE
+        # 1 2 3 6 have a saved copy; 5 has none but Telegram still hands it out
+        assert (plan.units, plan.held_saved, plan.live_restricted, plan.held_skipped, plan.service) == (5, 4, 1, 0, 1)
+        assert plan.media == 1  # post 3 is counted from its saved copy, not from what Telegram hands out now
+        # /repost copies the posts of the channel itself, post by post: a struck channel changes nothing about that
+        plain = await plan_repost(src, SRC, RepostOptions(), saved=lookup)
+        assert (plain.units, plain.held_saved, plain.live_restricted, plain.held_skipped) == (5, 0, 0, 0)
+        assert plain.channel_restricted == CHANNEL_NOTICE
+        await db.close()
+
+    run(go())
+
+
+# ========================================================================================= the read-back
+def test_every_copy_of_a_struck_channel_is_read_back_and_what_telegram_hides_is_counted(tmp_path):
+    async def go():
+        src, dst, db, dst_row, shift = await struck(tmp_path)
+        dst.restrict_new = lambda m: CHANNEL_NOTICE if m.media is not None else None  # the file of post 3 is blocked
+        res = await run_shift(src, db, shift, dst_row, 99, delay=0, settle_pause=0)
+        assert res.copied_units == 5 and res.failed == [] and res.held_back == []
+        assert res.checked == 5  # every copy of a restricted source
+        assert res.hidden == [(3, True)] and res.hidden_why == CHANNEL_NOTICE and res.dst_restricted is None
+        # the bot posted the saved text; the notice is Telegram's mark on the new post
+        assert dst.msgs[13].message == "Caption with link" and message_restriction(dst.msgs[13]) == CHANNEL_NOTICE
+        assert not any(NOTICE in (m.message or "") for m in dst.msgs.values())
+        await db.close()
+
+    run(go())
+
+
+def test_a_destination_that_is_restricted_itself_is_named(tmp_path):
+    async def go():
+        src, dst, db, dst_row, shift = await struck(tmp_path)
+        dst.restriction = CHANNEL_NOTICE
+        dst.restrict_new = lambda m: CHANNEL_NOTICE
+        res = await run_shift(src, db, shift, dst_row, 99, delay=0, settle_pause=0)
+        assert [i for i, _ in res.hidden] == [1, 2, 3] and res.dst_restricted == CHANNEL_NOTICE  # it stopped after 3
+        assert res.hidden_stop
+        await db.close()
+
+    run(go())
+
+
+def test_a_normal_channel_is_only_sampled_when_the_copies_are_read_back(tmp_path):
+    async def go():
+        src, dst, db, dst_row, shift = await held_setup(tmp_path, held=())
+        res = await run_shift(src, db, shift, dst_row, 99, delay=0, settle_pause=0)
+        assert res.copied_units == 5 and res.checked == CHECK_FIRST and res.hidden == [] and res.dst_restricted is None
+        assert not [r for r in dst.requests if isinstance(r, functions.channels.GetFullChannelRequest)]  # nothing to ask
+        await db.close()
+
+    run(go())
+
+
+def test_a_read_back_that_fails_does_not_fail_the_copy(tmp_path):
+    async def go():
+        for error in (rpc("ChannelPrivateError"), ConnectionError("the network dropped")):  # Telegram's refusal, or not
+            src, dst, db, dst_row, shift = await struck(tmp_path / type(error).__name__, last=1)  # post 1: no buttons to read back
+            dst.fail["GetMessagesRequest"] = error
+            res = await run_shift(src, db, shift, dst_row, 99, delay=0, settle_pause=0)
+            assert res.copied_units == 1 and res.failed == [] and res.checked == 0 and res.hidden == []
+            assert shown(dst) == {11: "Welcome"}
+            await db.close()
+
+    run(go())
+
+
+def test_a_source_that_cannot_be_asked_about_its_restriction_is_copied_as_an_unrestricted_one(tmp_path):
+    async def go():
+        for error in (rpc("ChannelPrivateError"), ConnectionError("the network dropped")):
+            src, dst, db, dst_row, shift = await held_setup(tmp_path / type(error).__name__, held=())
+            src.fail["GetFullChannelRequest"] = error
+            res = await run_shift(src, db, shift, dst_row, 99, delay=0, settle_pause=0)
+            assert res.copied_units == 5 and res.failed == [] and res.held_why is None and res.from_saved == 0
+            await db.close()
+
+    run(go())
+
+
+# ================================================================================================ the owner's view
+def test_the_plan_and_the_result_for_a_struck_channel_read_as_they_should(tmp_path):
+    async def go():
+        app, dst = await held_app(tmp_path, held=())
+        app.tg.restriction = CHANNEL_NOTICE
+        dst.fail["ForwardMessagesRequest"] = rpc("MessageIdInvalidError")
+        await app.text("/shift @srcch @dstch")
+        plan = norm(app.out.last_text)
+        assert "Telegram restricts Anime Source itself" in plan
+        assert "5 post(s) of Anime Source are restricted by Telegram" in plan and "copied from what My posts saved" in plan
+        assert "5 posts" in plan and "The new posts are read back" in plan
+
+        await app.press(app.out.callback_data("Start copying"))
+        done = norm(app.out.last_text)
+        assert "5 post(s) are restricted by Telegram" in done and "copied from what My posts saved of them" in done
+        assert "5 message(s) copied" in done and "hides" not in done
+        assert [m.message for i, m in sorted(dst.msgs.items()) if i > 10] == list(COPIES.values())
+        await app.db.close()
+
+    run(go())
+
+
+def test_the_report_says_when_telegram_itself_hides_the_new_posts(tmp_path):
+    async def go():
+        app, dst = await held_app(tmp_path, held=())
+        app.tg.restriction = CHANNEL_NOTICE
+        dst.restrict_new = lambda m: CHANNEL_NOTICE if m.media is not None else None
+        await app.text("/shift @srcch @dstch")
+        await app.press(app.out.callback_data("Start copying"))
+        done = norm(app.out.last_text)
+        assert "Telegram itself hides 1 of the 5 new post(s) I read back" in done
+        assert "carry its restriction mark" in done and "never posts one" in done
+        assert "1 post(s) of these carry a file" in done and "--no-media" in done
+        assert "The saved posts in My posts are intact" in done
+        assert "5 message(s) copied" in done  # the copies exist: Undo removes them
+        assert any("Undo" in t for row in app.out.last_buttons() for t in row)
+        await app.db.close()
+
+    run(go())
+
+
+def test_the_report_names_a_restricted_destination(tmp_path):
+    async def go():
+        app, dst = await held_app(tmp_path, held=())
+        app.tg.restriction = CHANNEL_NOTICE
+        dst.restriction = CHANNEL_NOTICE
+        dst.restrict_new = lambda m: CHANNEL_NOTICE
+        await app.text("/shift @srcch @dstch")
+        await app.press(app.out.callback_data("Start copying"))
+        done = norm(app.out.last_text)
+        assert "Telegram itself hides 3 of the 3 new post(s) I read back" in done
+        assert "Telegram restricts the destination channel itself" in done
+        await app.db.close()
+
+    run(go())
+
+
+def test_a_repost_plan_for_a_struck_channel_points_at_shift(tmp_path):
+    from .test_repost_flow import make_app
+
+    async def go():
+        app = await make_app(tmp_path, msgs={1: make_msg(1, "First post"), 2: make_msg(2, "Second post")})
+        app.tg.restriction = CHANNEL_NOTICE
+        await app.text("/repost --no-typed")
+        await app.press(app.out.callback_data("Test"))
+        plan = norm(app.out.last_text)
+        assert "Telegram restricts Test itself" in plan and "use /shift" in plan
+        assert "2 posts" in plan and "no saved copy" not in plan  # /repost does not look at My posts
+        await app.db.close()
+
+    run(go())
+
+
+# ====================================================================== stopping when Telegram hides the copies
+def test_a_shift_stops_by_itself_when_telegram_hides_every_new_copy(tmp_path):
+    async def go():
+        src, dst, db, dst_row, shift = await struck(tmp_path)
+        dst.restrict_new = lambda m: CHANNEL_NOTICE  # every new post is hidden
+        res = await run_shift(src, db, shift, dst_row, 99, delay=0, settle_pause=0)
+        assert res.stopped and res.hidden_stop and res.copied_units == 3  # not 5: it did not go on
+        assert [i for i, _ in res.hidden] == [1, 2, 3] and res.checked == 3
+        assert shown(dst) == {11: "Welcome", 12: "Episode 2 is out", 13: "Caption with link"}
+        assert res.phase == "copy" and res.final is None  # the finishing touches wait for the rest
+        await db.close()
+
+    run(go())
+
+
+def test_the_owner_can_go_on_anyway_and_it_does_not_stop_again(tmp_path):
+    async def go():
+        src, dst, db, dst_row, shift = await struck(tmp_path)
+        dst.restrict_new = lambda m: CHANNEL_NOTICE
+        first = await run_shift(src, db, shift, dst_row, 99, delay=0, settle_pause=0)
+        assert first.hidden_stop and first.copied_units == 3
+        await db.set_mark("s1", "hiddenok")  # what pressing Continue anyway rests on
+        again = await run_shift(src, db, shift, dst_row, 99, delay=0, settle_pause=0)
+        assert not again.stopped and not again.hidden_stop and again.copied_units == 2 and again.skipped_done == 3
+        assert shown(dst) == COPIES
+        await db.close()
+
+    run(go())
+
+
+def test_a_shift_does_not_stop_when_only_some_copies_are_hidden(tmp_path):
+    async def go():
+        src, dst, db, dst_row, shift = await struck(tmp_path)
+        dst.restrict_new = lambda m: CHANNEL_NOTICE if m.media is not None else None  # only the photo
+        res = await run_shift(src, db, shift, dst_row, 99, delay=0, settle_pause=0)
+        assert not res.stopped and res.copied_units == 5 and res.hidden == [(3, True)]
+        await db.close()
+
+    run(go())
+
+
+def test_a_shift_with_fewer_copies_than_the_threshold_is_not_stopped(tmp_path):
+    async def go():
+        src, dst, db, dst_row, shift = await struck(tmp_path, last=2)
+        dst.restrict_new = lambda m: CHANNEL_NOTICE
+        res = await run_shift(src, db, shift, dst_row, 99, delay=0, settle_pause=0)
+        assert not res.stopped and res.copied_units == 2 and len(res.hidden) == 2  # reported, nothing left to stop
+        await db.close()
+
+    run(go())
+
+
+def test_the_stopped_shift_explains_itself_and_continue_anyway_goes_on(tmp_path):
+    async def go():
+        app, dst = await held_app(tmp_path, held=())
+        app.tg.restriction = CHANNEL_NOTICE
+        dst.restrict_new = lambda m: CHANNEL_NOTICE
+        await app.text("/shift @srcch @dstch")
+        await app.press(app.out.callback_data("Start copying"))
+        done = norm(app.out.last_text)
+        assert "stopped" in done and "3 message(s) copied so far" in done
+        assert "I stopped after 3 post(s)" in done and "Telegram hides every new copy I read back" in done
+        assert "Telegram itself hides 3 of the 3 new post(s) I read back" in done
+        assert any("Continue" in t for t in buttons(app)) and any("Undo" in t for t in buttons(app))
+        sid = app.out.callback_data("Undo").split(":")[1]
+        assert await app.db.marks_of(sid) == {"hiddenok"}
+        assert len([i for i in dst.msgs if i > 10]) == 3  # three posts, not the whole channel
+
+        await app.press(app.out.callback_data("Continue"))
+        done = norm(app.out.last_text)
+        assert "5 message(s) copied" in done and "I stopped" not in done  # it went on, as the owner chose
+        assert len([i for i in dst.msgs if i > 10]) == 5
+        await app.db.close()
+
+    run(go())

tests/test_shift.py

@@ -390,6 +390,62 @@ def test_the_whole_flow_copy_then_undo(tmp_path):
     run(go())
 
 
+def test_without_arguments_a_finished_shift_can_still_be_undone(tmp_path):
+    """The result message of a finished shift may be far up the chat: /shift alone offers its Undo again."""
+    async def go():
+        app, dst = await shift_app(tmp_path)
+        before = len(app.out.log)
+        await app.text("/shift")  # nothing was shifted yet: only the usage
+        assert len(app.out.log) - before == 1 and "Usage: /shift" in norm(app.out.last_text)
+
+        await app.text("/shift @srcch @dstch")
+        await app.press(app.out.callback_data("Start copying"))
+        assert "7 message(s) copied" in norm(app.out.last_text)
+
+        await app.text("/shift")
+        assert "Usage: /shift" in norm(app.out.log[-2][1])  # the usage first, the finished shift right under it
+        text = norm(app.out.last_text)
+        assert "Latest finished shift - you can still undo it" in text and "7 message(s) copied" in text
+        await app.press(app.out.callback_data("Undo"))
+        await app.press(app.out.callback_data("Yes, remove"))
+        assert "7 copied message(s) removed from Anime Backup" in norm(app.out.last_text)
+        assert sorted(dst.msgs) == list(range(1, 11))
+
+        before = len(app.out.log)
+        await app.text("/shift")  # undone: nothing is left to offer
+        assert len(app.out.log) - before == 1 and "Latest finished shift" not in norm(app.out.last_text)
+        await app.db.close()
+
+    run(go())
+
+
+def test_only_a_finished_shift_that_still_has_copies_is_offered(tmp_path):
+    async def go():
+        app, dst = await shift_app(tmp_path)
+        await app.text("/shift @srcch @dstch")
+        await app.press(app.out.callback_data("Start copying"))
+        first = await app.db.last_finished_shift()
+        assert first is not None
+        # every copy of it was deleted some other way: nothing to undo, so nothing is offered
+        await app.db.mark_shift_deleted(first.id, 2, await app.db.shift_pending_new(first.id))
+        assert await app.db.last_finished_shift() is None
+        before = len(app.out.log)
+        await app.text("/shift")
+        assert len(app.out.log) - before == 1 and "Latest finished shift" not in norm(app.out.last_text)
+
+        # a newer shift is the one offered
+        await app.text("/shift @srcch @dstch")
+        await app.press(app.out.callback_data("Start copying"))
+        second = await app.db.last_finished_shift()
+        assert second is not None and second.id != first.id
+        await app.text("/shift")
+        assert "Latest finished shift" in norm(app.out.last_text)
+        assert app.out.callback_data("Undo") == f"sfu:{second.id}"
+        await app.db.close()
+
+    run(go())
+
+
 def test_a_destination_the_bot_does_not_know_yet_becomes_one_of_its_channels(tmp_path):
     async def go():
         app, dst = await shift_app(tmp_path)

tests/test_shift_extras.py

@@ -381,7 +381,7 @@ def test_an_unknown_option_is_explained(tmp_path):
     async def go():
         app, dst, other = await extras_app(tmp_path)
         await app.text("/shift @srcch @dstch -x")
-        assert "Unknown option -x" in norm(app.out.last_text) and "-c and -all" in norm(app.out.last_text)
+        assert "Unknown option -x" in norm(app.out.last_text) and "-c, -all and --no-media" in norm(app.out.last_text)
         assert dst.title == "Anime Backup"
         await app.db.close()