Automated sync from GitHub

.gitattributes

@@ -1,4 +1,3 @@
-*.ttf filter=lfs diff=lfs merge=lfs -text
 *.otf filter=lfs diff=lfs merge=lfs -text
 *.webp filter=lfs diff=lfs merge=lfs -text
 *.mp4 filter=lfs diff=lfs merge=lfs -text
@@ -6,3 +5,4 @@
 *.jpg filter=lfs diff=lfs merge=lfs -text
 *.jpeg filter=lfs diff=lfs merge=lfs -text
 *.png filter=lfs diff=lfs merge=lfs -text
+*.ttf filter=lfs diff=lfs merge=lfs -text

bot/app.py

@@ -21,7 +21,7 @@ from telethon import errors, events, functions, types
 from telethon.tl.custom import Button
 
 from . import fasttransfer as transfer
-from . import errorlog, media, mediainfo, panels, speed, sysinfo
+from . import errorlog, media, mediainfo, panels, richtext, speed, sysinfo
 from .config import Config
 from .db import MAX_FORMATS, Database
 from .errors import FormatError
@@ -30,7 +30,7 @@ from .parser import compute_new_name, missing_variables, parse_caption, parse_in
 from .pipeline import PROBE_EXTS, Renamer, embedded_thumb, original_name_of, source_ext
 from .queue import Job, JobQueue
 from .sequence import SeqItem, SequenceState, analyse, format_report
-from .settings_ops import META_COMMANDS, apply_setting, parse_channel, preview_template
+from .settings_ops import CLEAR_WORDS, META_COMMANDS, apply_setting, parse_channel, preview_template
 from .telegraph import Telegraph, TelegraphError
 from .utils import clip, esc, human_size, human_uptime
 
@@ -354,6 +354,17 @@ class App:
     def _args(event: Any) -> str:
         return (event.pattern_match.group(1) or "").strip() if event.pattern_match else ""
 
+    @staticmethod
+    def _caption_input(event: Any, start: int = 0) -> str:
+        """The caption the person sent (the message from ``start`` on).  Formatting they put on it in Telegram (bold, italic,
+        links...) becomes the HTML that is stored, so nobody has to type tags; without any formatting it is taken as typed."""
+        raw = event.raw_text or ""
+        typed = raw[start:]
+        if typed.strip().lower() in CLEAR_WORDS:  # "reset" stays a reset even if it was made bold
+            return typed
+        entities = getattr(getattr(event, "message", None), "entities", None)
+        return richtext.from_entities(raw, entities, start) or typed
+
     async def _reply(self, event: Any, text: str, rows: Optional[List[panels.Row]] = None) -> Any:
         buttons = panels.to_buttons(rows) if rows else None
         try:
@@ -402,7 +413,7 @@ class App:
 
     async def _edit_panel(self, event: Any, text: str, rows: List[panels.Row]) -> None:
         try:
-            await event.edit(text, buttons=panels.to_buttons(rows), parse_mode="html")
+            await event.edit(text, buttons=panels.to_buttons(rows), parse_mode="html", link_preview=False)
         except errors.MessageNotModifiedError:
             pass
         except errors.BotInlineDisabledError:  # inline mode was switched off in @BotFather: the Share button can't be shown
@@ -538,15 +549,17 @@ class App:
         arg = self._args(event)
         uid = event.sender_id
         if arg:
-            _ok, msg = await apply_setting(self.db, uid, "caption", arg)
+            _ok, msg = await apply_setting(self.db, uid, "caption", self._caption_input(event, event.pattern_match.start(1)))
             await self._reply(event, msg)
             return
         s = await self.db.get_settings(uid)
         await self._reply(
             event,
-            f"<b>Current caption template:</b>\n<code>{esc(s['caption'])}</code>\n\n"
+            f"<b>Current caption template:</b>\n{richtext.preview(s['caption'], 600)}\n\n"
             f"Variables: {panels.CAPTION_VARIABLES}\n"
-            "Change it: <code>/caption &lt;b&gt;{filename}&lt;/b&gt;</code> · restore: <code>/caption reset</code>",
+            "Change it: send <code>/caption</code> and your caption in one message. Format it the way you do in any message "
+            "(<b>bold</b>, <i>italic</i>, links…) and I'll keep it exactly like that.\n"
+            "Restore the default: <code>/caption reset</code>",
         )
 
     async def cmd_remove(self, event: Any) -> None:
@@ -1501,7 +1514,8 @@ class App:
         if p.kind == "resize:size":
             await self._on_resize_text(event, p)
             return
-        ok, msg = await apply_setting(self.db, uid, p.kind, event.raw_text)
+        value = self._caption_input(event) if p.kind == "caption" else event.raw_text
+        ok, msg = await apply_setting(self.db, uid, p.kind, value)
         if not ok:
             await self._reply(event, msg + "\n\nTry again, or tap Cancel.")
             return
@@ -1514,7 +1528,7 @@ class App:
         limit = 1000 if p.is_photo else 3900
         full = f"{msg}\n\n{text}" if visible_len(msg) + visible_len(text) < limit else f"{msg}\n\n(Open the menu to see the update.)"
         try:
-            await self.client.edit_message(p.chat_id, p.msg_id, full, buttons=panels.to_buttons(rows), parse_mode="html")
+            await self.client.edit_message(p.chat_id, p.msg_id, full, buttons=panels.to_buttons(rows), parse_mode="html", link_preview=False)
         except Exception:  # noqa: BLE001
             await self._reply(event, full, rows)
 
@@ -1732,7 +1746,7 @@ class App:
         elif action == "sh":  # Share format, shown while inline mode wasn't known to be on: just try the real button
             text, rows = panels.format_view(fmt, share_inline=True)
             try:
-                await event.edit(text, buttons=panels.to_buttons(rows), parse_mode="html")
+                await event.edit(text, buttons=panels.to_buttons(rows), parse_mode="html", link_preview=False)
             except errors.MessageNotModifiedError:
                 pass
             except errors.BotInlineDisabledError:  # Telegram refuses it: inline mode is off
@@ -1767,7 +1781,7 @@ class App:
     async def _on_format_text(self, event: Any, p: Pending) -> None:
         uid = event.sender_id
         field = p.kind[4:]
-        text = event.raw_text or ""
+        text = self._caption_input(event) if field == "caption" else event.raw_text or ""
         if field == "thumb":
             await self._reply(event, "Send a photo for the thumbnail, or tap Cancel.")
             return
@@ -1825,7 +1839,7 @@ class App:
         limit = 1000 if p.is_photo else 3900
         full = f"{msg}\n\n{text}" if visible_len(msg) + visible_len(text) < limit else f"{msg}\n\n(Open the format to see the update.)"
         try:
-            await self.client.edit_message(p.chat_id, p.msg_id, full, buttons=panels.to_buttons(rows), parse_mode="html")
+            await self.client.edit_message(p.chat_id, p.msg_id, full, buttons=panels.to_buttons(rows), parse_mode="html", link_preview=False)
         except Exception:  # noqa: BLE001
             await self._reply(event, full, rows)
 

bot/panels.py

@@ -8,6 +8,7 @@ from __future__ import annotations
 
 from typing import Any, Dict, Iterable, List, Sequence, Tuple
 
+from . import richtext
 from .db import DEFAULT_SETTINGS
 from .media import LANGUAGES, canonical_lang
 from .parser import CAPTION_VARS, TEMPLATE_VARS
@@ -120,9 +121,10 @@ def rename(s: Dict[str, Any]) -> Panel:
 def caption(s: Dict[str, Any], has_thumb: bool) -> Panel:
     text = (
         "<b>CAPTION &amp; THUMBNAIL</b>\n\n"
-        f"<b>Caption template:</b> {_code(s['caption'], 160)}\n"
+        f"<b>Caption template:</b> {richtext.preview(s['caption'], 160)}\n"
         f"Variables: {CAPTION_VARIABLES}\n"
-        "Telegram HTML works, e.g. <code>&lt;b&gt;{filename}&lt;/b&gt;</code>\n\n"
+        "Tap Set Caption and format it the way you do in any message (<b>bold</b>, <i>italic</i>, links…): "
+        "I keep it exactly like that.\n\n"
         f"<b>Thumbnail:</b> {'✅ saved' if has_thumb else '❌ none (a frame of the video is used)'}\n"
         "Send me a photo to set your thumbnail."
     )
@@ -292,7 +294,7 @@ def _format_lines(fmt: Dict[str, Any], *, thumb_yes: str, thumb_no: str) -> List
     m = fmt["meta"]
     lines = [
         f"{DOT} <b>Template</b> ▸ {_code(fmt['template'], 60)}",
-        f"{DOT} <b>Caption</b> ▸ {_code(fmt['caption'], 60)}" + ("" if _custom_caption(fmt) else " · default"),
+        f"{DOT} <b>Caption</b> ▸ {richtext.preview(fmt['caption'], 60)}" + ("" if _custom_caption(fmt) else " · default"),
         f"{DOT} <b>Thumbnail</b> ▸ " + (thumb_yes if fmt.get("has_thumb") else thumb_no),
     ]
     for key, label in FORMAT_FIELD_LABELS:
@@ -510,7 +512,8 @@ PROMPTS: Dict[str, str] = {
     "caption": (
         "<b>Send the new caption template.</b>\n\n"
         f"Variables: {CAPTION_VARIABLES}\n"
-        "Telegram HTML works (<code>&lt;b&gt;</code>, <code>&lt;i&gt;</code>, <code>&lt;code&gt;</code>)."
+        "Format it the way you do in any message (<b>bold</b>, <i>italic</i>, links…) and I'll keep it exactly like that. "
+        "No HTML needed."
     ),
     "remove": (
         "<b>Send the words to strip from names.</b>\n\n"
@@ -550,7 +553,8 @@ FORMAT_PROMPTS: Dict[str, str] = {
     "caption": (
         "<b>Send the caption for this format.</b>\n\n"
         f"Variables: {CAPTION_VARIABLES}\n"
-        "Telegram HTML works (<code>&lt;b&gt;</code>, <code>&lt;i&gt;</code>, <code>&lt;code&gt;</code>).\n"
+        "Format it the way you do in any message (<b>bold</b>, <i>italic</i>, links…) and I'll keep it exactly like that. "
+        "No HTML needed.\n"
         "Send <code>reset</code> to go back to the default: the new file name in bold."
     ),
     "title": PROMPTS["mt_title"], "author": PROMPTS["mt_author"], "artist": PROMPTS["mt_artist"],

bot/richtext.py

@@ -0,0 +1,161 @@
+"""Telegram formatting <-> the HTML a caption is stored as.
+
+People format a caption like any message: select words and make them bold, italic, a link...  Telegram delivers that
+as *entities* (kind, offset, length) next to the plain text.  Captions are kept as Telegram HTML (the variables are
+filled in and the HTML is sent with ``parse_mode="html"``), so ``from_entities`` turns the entities into that HTML and
+nobody has to type a tag, and ``preview`` turns stored HTML back into something that is safe to show in a panel.
+
+Entity offsets count UTF-16 code units, so the text is handled the way Telethon does it: an emoji outside the basic
+plane is two "surrogate" characters while the work is going on.
+"""
+from __future__ import annotations
+
+import html
+import re
+from typing import Any, Dict, Iterable, List, NamedTuple, Optional, Tuple
+
+from telethon.extensions import html as telegram_html
+from telethon.helpers import add_surrogate, del_surrogate
+from telethon.tl import types
+
+from .parser import CAPTION_VARS
+from .utils import esc
+
+_VARIABLE = re.compile(r"\{(?:%s)\}" % "|".join(map(re.escape, CAPTION_VARS)))
+
+_SIMPLE = {
+    types.MessageEntityBold: ("<b>", "</b>"),
+    types.MessageEntityItalic: ("<i>", "</i>"),
+    types.MessageEntityUnderline: ("<u>", "</u>"),
+    types.MessageEntityStrike: ("<s>", "</s>"),
+    types.MessageEntityCode: ("<code>", "</code>"),
+}
+
+
+class Span(NamedTuple):
+    start: int
+    end: int
+    open: str
+    close: str
+
+
+def _tags(entity: Any) -> Optional[Tuple[str, str]]:
+    """The HTML tags for one entity.  ``None`` for the ones that are not formatting the bot keeps: what Telegram finds by
+    itself (links in the text, @mentions, #hashtags, /commands) comes back by itself when the caption is sent.  Spoilers and
+    custom emoji are not kept either: Telethon's HTML has no spoiler, and a bot may not send custom emoji."""
+    kind = type(entity)
+    if kind in _SIMPLE:
+        return _SIMPLE[kind]
+    if kind is types.MessageEntityTextUrl:
+        return f'<a href="{html.escape(entity.url, quote=True)}">', "</a>"
+    if kind is types.MessageEntityMentionName:
+        return f'<a href="tg://user?id={entity.user_id}">', "</a>"
+    if kind is types.MessageEntityPre:
+        language = html.escape(getattr(entity, "language", "") or "", quote=True)
+        return (f'<pre><code class="language-{language}">', "</code></pre>") if language else ("<pre>", "</pre>")
+    if kind is types.MessageEntityBlockquote:
+        return ("<blockquote expandable>" if getattr(entity, "collapsed", False) else "<blockquote>"), "</blockquote>"
+    return None
+
+
+def _whole(body: str, at: int) -> int:
+    """``at``, moved off the middle of a surrogate pair."""
+    if 0 < at < len(body) and "\udc00" <= body[at] <= "\udfff" and "\ud800" <= body[at - 1] <= "\udbff":
+        return at + 1
+    return at
+
+
+def _tidy(spans: Iterable[Span]) -> List[Span]:
+    """Spans of one kind never overlap afterwards.  The HTML parser can't nest a tag in itself (``<b>a<b>b</b>c</b>`` loses
+    its outer one), so identical spans that overlap or touch are joined, and of two links (or other tags with different
+    details) that overlap, the one that starts later only gets what the first leaves."""
+    groups: Dict[str, List[Span]] = {}
+    for s in spans:
+        groups.setdefault(re.match(r"<(\w+)", s.open).group(1), []).append(s)
+    out: List[Span] = []
+    for group in groups.values():
+        kept: List[Span] = []
+        for s in sorted(group, key=lambda s: (s.start, -s.end)):
+            if kept and s.start <= kept[-1].end:
+                last = kept[-1]
+                if s.open == last.open:
+                    kept[-1] = last._replace(end=max(last.end, s.end))
+                    continue
+                if s.start < last.end:
+                    s = s._replace(start=last.end)
+                    if s.end <= s.start:
+                        continue
+            kept.append(s)
+        out += kept
+    return out
+
+
+def _render(body: str, spans: Iterable[Span]) -> str:
+    """``body`` as HTML with the spans' tags in it.  Spans that overlap without one containing the other are split, so
+    that the tags always nest."""
+    spans = [s._replace(start=_whole(body, s.start), end=_whole(body, s.end)) for s in spans]
+    spans = _tidy(s for s in spans if s.end > s.start)
+    points = sorted({0, len(body)} | {p for s in spans for p in (s.start, s.end)})
+    out: List[str] = []
+    running: List[Span] = []  # the spans that are open right now, outermost first
+    for index, at in enumerate(points):
+        ending = [s for s in running if s.end == at]
+        if ending:
+            depth = min(running.index(s) for s in ending)
+            inner, running = running[depth:], running[:depth]
+            out += [s.close for s in reversed(inner)]
+            for s in inner:
+                if s.end != at:  # it goes on past this point: close it around the one that ended and open it again
+                    out.append(s.open)
+                    running.append(s)
+        for s in sorted((s for s in spans if s.start == at), key=lambda s: -s.end):  # on a tie the earlier one stays outside
+            out.append(s.open)
+            running.append(s)
+        if index + 1 < len(points):
+            out.append(esc(body[at:points[index + 1]]))
+    return "".join(out)
+
+
+def from_entities(text: str, entities: Optional[Iterable[Any]], start: int = 0) -> Optional[str]:
+    """``text[start:]`` as HTML that carries the formatting in ``entities``, or ``None`` when there is none to keep.
+
+    ``start`` skips what comes before the caption (a ``/caption`` command).  A ``{variable}`` is never split: a format that
+    covers only part of one (double-tapping the word inside the braces does that) is widened to the whole variable, because
+    the variable is replaced as a whole.
+    """
+    if not text or not entities:
+        return None
+    first = len(add_surrogate(text[:start]))
+    body = add_surrogate(text)[first:]
+    first += len(body) - len(body.lstrip())
+    body = body.strip()
+    variables = [(m.start(), m.end()) for m in _VARIABLE.finditer(body)]
+    spans = []
+    for entity in entities:
+        tags = _tags(entity)
+        if tags is None:
+            continue
+        a, b = max(entity.offset - first, 0), min(entity.offset + entity.length - first, len(body))
+        if b <= a:
+            continue
+        for lo, hi in variables:
+            if a < hi and b > lo:
+                a, b = min(a, lo), max(b, hi)
+        spans.append(Span(a, b, *tags))
+    return del_surrogate(_render(body, spans)) if spans else None
+
+
+def preview(template: str, limit: int) -> str:
+    """A stored caption template as HTML that is safe to show in a panel: its formatting is shown, its tags are balanced
+    whatever was typed, and it is cut to ``limit`` characters."""
+    text, entities = telegram_html.parse(template or "")
+    if not text:
+        return "—"
+    body = add_surrogate(text)
+    cut = len(body) if len(body) <= limit else _whole(body, limit - 1)
+    spans = []
+    for entity in entities:
+        tags = _tags(entity)
+        if tags is not None:
+            spans.append(Span(entity.offset, min(entity.offset + entity.length, cut), *tags))
+    return del_surrogate(_render(body[:cut], spans)) + ("…" if cut < len(body) else "")

bot/settings_ops.py

@@ -66,7 +66,7 @@ async def apply_setting(db: Database, uid: int, kind: str, text: str) -> Tuple[b
         if _is_clear(text):
             text = DEFAULT_SETTINGS["caption"]
         if len(text) > 800:
-            return False, "❌ That caption is too long (800 characters max)."
+            return False, "❌ That caption is too long (800 characters max, formatting and link addresses included)."
         unknown = sorted(set(_TOKEN_RE.findall(text)) - set(CAPTION_VARS))
         await db.update_settings(uid, caption=text)
         msg = "✅ Caption template saved."

tests/appkit.py

@@ -9,7 +9,7 @@ from bot.app import App
 from bot.config import load_config
 from bot.db import Database
 
-from .tgfakes import FakeCallback, FakeClient, FakeEvent, dispatch, document_message, flat_buttons
+from .tgfakes import FakeCallback, FakeClient, FakeEvent, dispatch, document_message, flat_buttons, telegram_text
 
 OWNER, ADMIN_CFG, ADMIN_DB, STRANGER = 1000001, 1000002, 1000003, 1999999
 LOG = -1001111111111
@@ -44,6 +44,12 @@ async def say(ctx, sender, text="", **kw):
     return ev
 
 
+async def say_formatted(ctx, sender, markup, **kw):
+    """Send a message with formatting, written like ``"[b]bold[/b] [i]italic[/i] [a=https://x.io]link[/a]"`` (see ``telegram_text``)."""
+    text, entities = telegram_text(markup)
+    return await say(ctx, sender, text, entities=entities, **kw)
+
+
 async def send_doc(ctx, sender, name=EPISODE, size=50 * 1024 * 1024, **kw):
     message = document_message(name, size, **kw)
     ctx.client.messages[message.id] = message  # so get_messages() can find it again

tests/test_caption_formatting_app.py

@@ -0,0 +1,217 @@
+"""A caption keeps the formatting it was sent with (bold, italic, links...): nobody has to type HTML."""
+from telethon.extensions import html as telegram_html
+
+from bot.pipeline import make_caption
+
+from .appkit import OWNER, click, data_of, prefs, say, say_formatted
+from .test_formats_app import make_format
+
+FORMATTED = "[b]Watch[/b] {title} [i]episode {episode}[/i] on [a=https://t.me/Anime_X]our channel[/a]"
+STORED = '<b>Watch</b> {title} <i>episode {episode}</i> on <a href="https://t.me/Anime_X">our channel</a>'
+
+
+# ---------------------------------------------------------------------------
+# /caption
+# ---------------------------------------------------------------------------
+
+
+async def test_the_caption_command_keeps_the_formatting_that_was_applied(ctx):
+    ev = await say_formatted(ctx, OWNER, "/caption " + FORMATTED)
+    assert (await prefs(ctx))["caption"] == STORED
+    assert "Caption template saved" in ev.texts[0] and "Unknown" not in ev.texts[0]
+
+
+async def test_the_caption_command_shows_the_caption_with_its_formatting(ctx):
+    await say_formatted(ctx, OWNER, "/caption " + FORMATTED)
+    text = (await say(ctx, OWNER, "/caption")).texts[0]
+    assert "Current caption template" in text and STORED in text  # shown as the person wrote it, not as tags to read
+    assert "&lt;b&gt;" not in text and "&lt;a href" not in text
+
+
+async def test_the_command_may_carry_the_bot_name_extra_spaces_and_several_lines(ctx):
+    await say_formatted(ctx, OWNER, "/caption@RenameBot    [b]{filename}[/b]\nsecond [i]line[/i] {size}")
+    assert (await prefs(ctx))["caption"] == "<b>{filename}</b>\nsecond <i>line</i> {size}"
+
+
+async def test_emoji_before_the_formatted_words_do_not_shift_it(ctx):
+    await say_formatted(ctx, OWNER, "/caption 🔥🔥 [b]{filename}[/b] 👨‍👩‍👧 [i]{size}[/i]")
+    assert (await prefs(ctx))["caption"] == "🔥🔥 <b>{filename}</b> 👨‍👩‍👧 <i>{size}</i>"
+
+
+async def test_html_typed_by_hand_still_works_when_nothing_is_formatted(ctx):
+    await say(ctx, OWNER, "/caption <i>{filename}</i> · <a href=\"https://t.me/x\">join</a>")
+    assert (await prefs(ctx))["caption"] == '<i>{filename}</i> · <a href="https://t.me/x">join</a>'
+
+
+async def test_what_telegram_finds_by_itself_is_not_formatting(ctx):
+    """A #hashtag or an @mention is an entity too, but nothing was formatted: typed tags are still taken as typed."""
+    await say_formatted(ctx, OWNER, "/caption <i>{filename}</i> [tag]#anime[/tag] [at]@channel[/at]")
+    assert (await prefs(ctx))["caption"] == "<i>{filename}</i> #anime @channel"
+
+
+async def test_once_words_are_formatted_the_message_is_taken_as_written(ctx):
+    await say_formatted(ctx, OWNER, "/caption <b>{filename}</b> [i]x[/i]")
+    assert (await prefs(ctx))["caption"] == "&lt;b&gt;{filename}&lt;/b&gt; <i>x</i>"  # the tags the person typed are just text
+
+
+async def test_a_spoiler_is_not_kept_so_its_words_stay_plain(ctx):
+    await say_formatted(ctx, OWNER, "/caption [spoiler]secret[/spoiler] {title}")
+    assert (await prefs(ctx))["caption"] == "secret {title}"
+
+
+async def test_reset_still_resets_when_it_was_made_bold(ctx):
+    await say(ctx, OWNER, "/caption custom {filename}")
+    ev = await say_formatted(ctx, OWNER, "/caption [b]reset[/b]")
+    assert (await prefs(ctx))["caption"] == "<b>{filename}</b>" and "Caption template saved" in ev.texts[0]
+
+
+async def test_a_variable_cut_by_the_formatting_is_still_a_whole_variable(ctx):
+    """Double-tapping {filename} selects the word without its braces: the bold covers the whole variable all the same."""
+    ev = await say_formatted(ctx, OWNER, "/caption {[b]filename[/b]} · {siz[i]e} left[/i]")
+    assert (await prefs(ctx))["caption"] == "<b>{filename}</b> · <i>{size} left</i>"
+    assert "Unknown" not in ev.texts[0]
+
+
+async def test_unknown_variables_are_still_reported(ctx):
+    ev = await say_formatted(ctx, OWNER, "/caption [b]{filename}[/b] {nonsense}")
+    assert "Unknown variables" in ev.texts[0] and "{nonsense}" in ev.texts[0]
+    assert (await prefs(ctx))["caption"] == "<b>{filename}</b> {nonsense}"
+
+
+async def test_a_caption_that_is_too_long_with_its_link_addresses_is_refused(ctx):
+    ev = await say_formatted(ctx, OWNER, f"/caption [a=https://example.com/{'a' * 800}]link[/a] {{filename}}")
+    assert "too long" in ev.texts[0] and "link addresses" in ev.texts[0]
+    assert (await prefs(ctx))["caption"] == "<b>{filename}</b>"
+
+
+# ---------------------------------------------------------------------------
+# the Set Caption button, then the caption
+# ---------------------------------------------------------------------------
+
+
+async def test_the_set_caption_prompt_keeps_the_formatting(ctx):
+    cb = await click(ctx, OWNER, "in:caption", msg_id=500)
+    assert "No HTML needed" in cb.edits[-1][0]
+    ev = await say_formatted(ctx, OWNER, FORMATTED, mid=60)
+    assert ev.deleted and OWNER not in ctx.app.pending
+    assert (await prefs(ctx))["caption"] == STORED
+    (chat, text, kw), = ctx.client.edited
+    assert kw["message"] == 500 and "Caption template saved" in text and "CAPTION" in text
+    assert STORED in text and "&lt;b&gt;" not in text  # the panel shows the caption formatted
+    assert "in:caption" in data_of(kw["buttons"])
+
+
+async def test_the_set_caption_prompt_still_takes_typed_html_and_reset(ctx):
+    await click(ctx, OWNER, "in:caption")
+    await say(ctx, OWNER, "<u>{filename}</u> {size}")
+    assert (await prefs(ctx))["caption"] == "<u>{filename}</u> {size}"
+    await click(ctx, OWNER, "in:caption")
+    await say_formatted(ctx, OWNER, "[b]reset[/b]")
+    assert (await prefs(ctx))["caption"] == "<b>{filename}</b>"
+
+
+async def test_formatting_sent_to_another_prompt_is_not_turned_into_html(ctx):
+    """Only captions are rich text: a template or a title is plain text, whatever it was formatted with."""
+    await click(ctx, OWNER, "in:template")
+    await say_formatted(ctx, OWNER, "[b]{title}[/b] [i]{quality}[/i]")
+    assert (await prefs(ctx))["template"] == "{title} {quality}"
+
+
+async def test_a_prompt_that_is_refused_stays_open_for_another_try(ctx):
+    await click(ctx, OWNER, "in:caption")
+    ev = await say_formatted(ctx, OWNER, f"[a=https://example.com/{'a' * 800}]link[/a]")
+    assert "too long" in ev.texts[0] and "Try again" in ev.texts[0] and OWNER in ctx.app.pending
+    await say_formatted(ctx, OWNER, "[b]{filename}[/b] ok")
+    assert OWNER not in ctx.app.pending and (await prefs(ctx))["caption"] == "<b>{filename}</b> ok"
+
+
+# ---------------------------------------------------------------------------
+# a saved format's own caption
+# ---------------------------------------------------------------------------
+
+
+async def test_a_formats_caption_prompt_keeps_the_formatting(ctx):
+    await ctx.db.update_settings(OWNER, caption="<u>{filename}</u>")
+    fmt = await make_format(ctx)
+    i = fmt["id"]
+    cb = await click(ctx, OWNER, f"fm:in:{i}:caption", msg_id=710)
+    assert "No HTML needed" in cb.edits[-1][0]
+    ev = await say_formatted(ctx, OWNER, FORMATTED)
+    assert ev.deleted and OWNER not in ctx.app.pending
+    assert (await ctx.db.get_format(OWNER, i))["caption"] == STORED
+    assert (await prefs(ctx))["caption"] == "<u>{filename}</u>"  # the user's own caption was never touched
+    chat, text, kw = ctx.client.edited[-1]
+    assert kw["message"] == 710 and "Caption template saved" in text
+    assert "<b>Caption</b> ▸ " + STORED in text and "· default" not in text  # the format's screen shows it formatted
+    assert f"fm:cr:{i}" in data_of(kw["buttons"])
+
+
+async def test_a_formats_caption_made_bold_exactly_like_the_default_counts_as_the_default(ctx):
+    fmt = await make_format(ctx, caption="<u>{filename}</u>")
+    await click(ctx, OWNER, f"fm:in:{fmt['id']}:caption")
+    await say_formatted(ctx, OWNER, "[b]{filename}[/b]")
+    chat, text, kw = ctx.client.edited[-1]
+    assert (await ctx.db.get_format(OWNER, fmt["id"]))["caption"] == "<b>{filename}</b>"
+    assert "· default" in text and f"fm:cr:{fmt['id']}" not in data_of(kw["buttons"])
+
+
+async def test_a_format_caption_reset_works_in_bold_and_other_fields_stay_plain(ctx):
+    fmt = await make_format(ctx, caption="<u>{filename}</u>")
+    i = fmt["id"]
+    await click(ctx, OWNER, f"fm:in:{i}:caption")
+    await say_formatted(ctx, OWNER, "[b]reset[/b]")
+    assert (await ctx.db.get_format(OWNER, i))["caption"] == "<b>{filename}</b>"
+    await click(ctx, OWNER, f"fm:in:{i}:title")
+    await say_formatted(ctx, OWNER, "[b]One Piece[/b]")
+    assert (await ctx.db.get_format(OWNER, i))["meta"]["title"] == "One Piece"  # metadata is not rich text
+
+
+# ---------------------------------------------------------------------------
+# a link in the caption must not put a link preview on the settings screens
+# ---------------------------------------------------------------------------
+
+
+async def test_a_link_in_a_caption_gets_no_link_preview_on_any_screen_that_shows_it(ctx):
+    await say_formatted(ctx, OWNER, "/caption " + FORMATTED)
+    cb = await click(ctx, OWNER, "m:cap")
+    assert STORED in cb.edits[-1][0] and cb.edit_kwargs[-1]["link_preview"] is False
+
+    await click(ctx, OWNER, "in:caption", msg_id=500)
+    await say_formatted(ctx, OWNER, FORMATTED)
+    assert STORED in ctx.client.edited[-1][1] and ctx.client.edited[-1][2]["link_preview"] is False
+
+    fmt = await make_format(ctx, caption=STORED)
+    cb = await click(ctx, OWNER, f"fm:v:{fmt['id']}")
+    assert STORED in cb.edits[-1][0] and cb.edit_kwargs[-1]["link_preview"] is False
+    await click(ctx, OWNER, f"fm:in:{fmt['id']}:caption", msg_id=710)
+    await say_formatted(ctx, OWNER, FORMATTED)
+    assert STORED in ctx.client.edited[-1][1] and ctx.client.edited[-1][2]["link_preview"] is False
+    cb = await click(ctx, OWNER, f"fm:sh:{fmt['id']}")
+    assert cb.edit_kwargs and all(kw.get("link_preview") is False for kw in cb.edit_kwargs)
+
+
+# ---------------------------------------------------------------------------
+# what the renamed file gets
+# ---------------------------------------------------------------------------
+
+
+async def test_the_formatting_follows_the_values_that_replace_the_variables(ctx):
+    await say_formatted(ctx, OWNER, "/caption [b]{filename}[/b] · [i]{size}[/i] · [a=https://t.me/x]{original}[/a]")
+    caption = make_caption((await prefs(ctx))["caption"], new_name="One Piece - E5.mkv", orig_name="1147 - One Piece [Dual].mkv",
+                           ext=".mkv", size=1536, duration=0)
+    text, entities = telegram_html.parse(caption)  # what Telegram makes of it when it is sent with parse_mode="html"
+    assert text == "One Piece - E5 · 1.50 KB · 1147 - One Piece [Dual].mkv"
+    shown = {type(e).__name__: (text[e.offset:e.offset + e.length], getattr(e, "url", None)) for e in entities}
+    assert shown == {
+        "MessageEntityBold": ("One Piece - E5", None),
+        "MessageEntityItalic": ("1.50 KB", None),
+        "MessageEntityTextUrl": ("1147 - One Piece [Dual].mkv", "https://t.me/x"),
+    }
+
+
+async def test_a_name_with_html_characters_cannot_break_the_formatting(ctx):
+    await say_formatted(ctx, OWNER, "/caption [b]{filename}[/b] [i]{original}[/i]")
+    caption = make_caption((await prefs(ctx))["caption"], new_name="Tom & <Jerry>.mkv", orig_name="a<b>&c.mkv", ext=".mkv", size=1, duration=0)
+    text, entities = telegram_html.parse(caption)
+    assert text == "Tom & <Jerry> a<b>&c.mkv"
+    assert [(type(e).__name__, e.offset, e.length) for e in entities] == [("MessageEntityBold", 0, 13), ("MessageEntityItalic", 14, 10)]

tests/test_formats_app.py

@@ -101,7 +101,7 @@ async def test_the_format_screen_shows_every_field_and_has_a_button_for_each(ctx
         assert f"fm:in:{i}:{field}" in data, field
     assert {f"fm:th:{i}", f"fm:del:{i}", "m:formats", "x:close"} <= set(data)
     assert f"fm:thd:{i}" not in data  # nothing to delete yet
-    assert f"fm:cr:{i}" not in data and "<b>Caption</b> ▸ <code>&lt;b&gt;{filename}&lt;/b&gt;</code> · default" in text  # nor to reset
+    assert f"fm:cr:{i}" not in data and "<b>Caption</b> ▸ <b>{filename}</b> · default" in text  # nor to reset
 
 
 async def test_editing_each_field_goes_through_the_same_checks_as_the_settings(ctx):
@@ -169,7 +169,7 @@ async def test_a_formats_caption_is_its_own_and_can_be_reset(ctx):
     assert ev.deleted and OWNER not in ctx.app.pending
     chat, text, kw = ctx.client.edited[-1]
     assert kw["message"] == 710 and "Caption template saved" in text and "Unknown variables" in text and "{nonsense}" in text
-    assert "<b>Caption</b> ▸ <code>&lt;b&gt;{filename}&lt;/b&gt; · {size} · {nonsense}</code>" in text and "· default" not in text
+    assert "<b>Caption</b> ▸ <b>{filename}</b> · {size} · {nonsense}" in text and "· default" not in text
     assert f"fm:cr:{i}" in data_of(kw["buttons"])  # a custom caption can be reset with one tap
 
     cb = await click(ctx, OWNER, f"fm:cr:{i}")

tests/test_panels.py

@@ -228,11 +228,11 @@ def test_known_languages_lists_names():
 
 def test_the_format_screen_shows_its_caption_and_offers_to_reset_a_custom_one():
     plain = panels.format_view(sample_format())
-    assert "<b>Caption</b> ▸ <code>&lt;b&gt;{filename}&lt;/b&gt;</code> · default" in plain[0]
+    assert "<b>Caption</b> ▸ <b>{filename}</b> · default" in plain[0]
     assert "fm:in:12:caption" in [d for row in plain[1] for _l, d in row] and "fm:cr:12" not in [d for row in plain[1] for _l, d in row]
 
     custom = panels.format_view(sample_format(caption="<i>{filename}</i> · {size}"))
-    assert "<b>Caption</b> ▸ <code>&lt;i&gt;{filename}&lt;/i&gt; · {size}</code>" in custom[0] and "default" not in custom[0].split("Thumbnail")[0]
+    assert "<b>Caption</b> ▸ <i>{filename}</i> · {size}" in custom[0] and "default" not in custom[0].split("Thumbnail")[0]
     assert ("↩ Reset Caption", "fm:cr:12") in [button for row in custom[1] for button in row]
     assert panels.without_inline(custom[1]) == panels.format_view(sample_format(caption="<i>{filename}</i> · {size}"), share_inline=False)[1]
     assert "caption" in panels.FORMAT_PROMPTS["caption"].lower() and "<code>reset</code>" in panels.FORMAT_PROMPTS["caption"]

tests/test_richtext.py

@@ -0,0 +1,253 @@
+"""Turning the formatting a person puts on a caption into the HTML that is stored, and showing it back."""
+import random
+
+import pytest
+from telethon.extensions import html as telegram_html
+from telethon.helpers import add_surrogate, del_surrogate
+from telethon.tl import types
+
+from bot.richtext import from_entities, preview
+
+from .tgfakes import telegram_text
+
+
+def html_of(markup, start=0):
+    text, entities = telegram_text(markup)
+    return from_entities(text, entities, start)
+
+
+def formats_of(html_text):
+    """What Telegram would show for stored HTML: ``{(kind, offset, length, url)}``, from Telethon's own parser."""
+    _text, entities = telegram_html.parse(html_text)
+    return {(type(e).__name__, e.offset, e.length, getattr(e, "url", None)) for e in entities}
+
+
+# ---------------------------------------------------------------------------
+# the formatting that is kept
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.parametrize("markup, expected", [
+    ("[b]Tokyo[/b] Revengers", "<b>Tokyo</b> Revengers"),
+    ("a [i]b[/i] c", "a <i>b</i> c"),
+    ("a [u]b[/u] c", "a <u>b</u> c"),
+    ("a [s]b[/s] c", "a <s>b</s> c"),
+    ("a [code]b[/code] c", "a <code>b</code> c"),
+    ("a [a=https://t.me/x]b[/a] c", 'a <a href="https://t.me/x">b</a> c'),
+    ("a [quote]b[/quote]", "a <blockquote>b</blockquote>"),
+    ("[pre=python]print(1)[/pre]", '<pre><code class="language-python">print(1)</code></pre>'),
+    ("[pre]x = 1[/pre]", "<pre>x = 1</pre>"),
+])
+def test_each_kind_of_formatting_becomes_its_tag(markup, expected):
+    assert html_of(markup) == expected
+
+
+def test_a_collapsed_quote_stays_collapsed():
+    entity = types.MessageEntityBlockquote(0, 3, collapsed=True)
+    assert from_entities("one", [entity]) == "<blockquote expandable>one</blockquote>"
+
+
+def test_a_mention_of_a_person_keeps_its_link():
+    assert from_entities("hi Sam", [types.MessageEntityMentionName(3, 3, 12345)]) == 'hi <a href="tg://user?id=12345">Sam</a>'
+
+
+def test_bold_italic_and_a_link_in_one_caption():
+    markup = "[b]Watch[/b] {title} [i]episode {episode}[/i] on [a=https://t.me/Anime_X]our channel[/a]"
+    assert html_of(markup) == '<b>Watch</b> {title} <i>episode {episode}</i> on <a href="https://t.me/Anime_X">our channel</a>'
+
+
+def test_formatting_that_is_not_the_persons_own_is_left_out():
+    """What Telegram finds by itself comes back by itself when the caption is sent; spoilers and custom emoji are not kept."""
+    assert html_of("[tag]#anime[/tag] [at]@channel[/at] [spoiler]secret[/spoiler] /start") is None
+    plain, entities = telegram_text("link https://example.com mail a@b.co")
+    entities += [types.MessageEntityUrl(5, 19), types.MessageEntityEmail(30, 6), types.MessageEntityCustomEmoji(0, 1, 5)]
+    assert from_entities(plain, entities) is None
+    assert html_of("[spoiler]a[/spoiler] [b]b[/b] [tag]#c[/tag]") == "a <b>b</b> #c"
+
+
+@pytest.mark.parametrize("text, entities", [("just words", None), ("just words", []), ("", [types.MessageEntityBold(0, 1)])])
+def test_no_formatting_means_nothing_to_convert(text, entities):
+    assert from_entities(text, entities) is None
+
+
+def test_text_that_looks_like_html_is_escaped_once_formatting_is_used():
+    assert html_of("[b]Tom & Jerry[/b] <3 > <i>not a tag</i>") == "<b>Tom &amp; Jerry</b> &lt;3 &gt; &lt;i&gt;not a tag&lt;/i&gt;"
+
+
+def test_a_link_address_survives_the_html():
+    url = 'https://example.com/?a=1&copy=2&q="z"'
+    stored = html_of(f"see [a={url}]this[/a]")
+    assert "&amp;copy=2" in stored  # written out, not read as the © character
+    assert formats_of(stored) == {("MessageEntityTextUrl", 4, 4, url)}
+
+
+def test_emoji_do_not_move_the_formatting():
+    """Telegram counts an emoji outside the basic plane (and each part of a family emoji) as two units."""
+    assert html_of("🔥🔥 [b]Hello[/b] 👨‍👩‍👧 [i]x[/i]") == "🔥🔥 <b>Hello</b> 👨‍👩‍👧 <i>x</i>"
+
+
+def test_the_command_in_front_is_skipped():
+    text, entities = telegram_text("/caption [b]Hi[/b] there")
+    assert from_entities(text, entities, start=len("/caption ")) == "<b>Hi</b> there"
+    whole = types.MessageEntityBold(0, len("/caption Hi"))  # formatting that began inside the command is cut at the caption
+    assert from_entities("/caption Hi", [whole], start=9) == "<b>Hi</b>"
+
+
+def test_start_counts_characters_not_utf16_units():
+    text, entities = telegram_text("🔥 hi [b]x[/b]")
+    assert from_entities(text, entities, start=2) == "hi <b>x</b>"
+
+
+def test_spaces_around_the_caption_are_dropped_with_their_formatting():
+    assert html_of("  [b]Hi[/b]  ") == "<b>Hi</b>"
+    assert html_of("[b]Hi  [/b]") == "<b>Hi</b>"
+    assert html_of("[b]   [/b]") is None
+
+
+# ---------------------------------------------------------------------------
+# formats that meet
+# ---------------------------------------------------------------------------
+
+
+def test_nested_formatting_stays_nested():
+    assert html_of("[b]a [i]b[/i] c[/b]") == "<b>a <i>b</i> c</b>"
+
+
+def test_formats_that_only_partly_overlap_are_split_so_the_tags_nest():
+    stored = html_of("[b]Hel[i]lo[/b] wo[/i]rld")
+    assert stored == "<b>Hel<i>lo</i></b><i> wo</i>rld"
+    # the same words are bold and italic as before
+    text, entities = telegram_html.parse(stored)
+    units = add_surrogate(text)
+    bold = "".join(units[e.offset:e.offset + e.length] for e in entities if isinstance(e, types.MessageEntityBold))
+    italic = "".join(units[e.offset:e.offset + e.length] for e in entities if isinstance(e, types.MessageEntityItalic))
+    assert (bold, italic) == ("Hello", "lo wo")
+
+
+def test_two_kinds_on_the_same_words():
+    stored = html_of("[b][i]x[/i][/b] y")
+    assert formats_of(stored) == {("MessageEntityBold", 0, 1, None), ("MessageEntityItalic", 0, 1, None)}
+
+
+def test_the_same_kind_twice_over_the_same_words_is_one_tag():
+    """The HTML parser can't nest a tag in itself, so ``<b>a<b>b</b>c</b>`` would lose its outer bold."""
+    text, entities = "abcde", [types.MessageEntityBold(0, 3), types.MessageEntityBold(1, 3), types.MessageEntityBold(4, 1)]
+    stored = from_entities(text, entities)
+    assert stored == "<b>abcde</b>"
+    assert formats_of(stored) == {("MessageEntityBold", 0, 5, None)}
+
+
+def test_of_two_overlapping_links_the_later_one_gets_what_is_left():
+    entities = [types.MessageEntityTextUrl(0, 5, "https://a.io"), types.MessageEntityTextUrl(3, 5, "https://b.io")]
+    assert from_entities("abcdefgh", entities) == '<a href="https://a.io">abcde</a><a href="https://b.io">fgh</a>'
+
+
+# ---------------------------------------------------------------------------
+# {variables} are replaced as a whole, so a format must not cut one in two
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.parametrize("markup, expected", [
+    ("Episode {[b]episode[/b]} here", "Episode <b>{episode}</b> here"),  # double-tapping the word leaves the braces out
+    ("[b]{epis[/b]ode} here", "<b>{episode}</b> here"),
+    ("Episode [b]{epis[/b]ode} here", "Episode <b>{episode}</b> here"),
+    ("Ep {epis[b]ode} here[/b]", "Ep <b>{episode} here</b>"),
+    ("[b]{title} {epis[/b]ode}", "<b>{title} {episode}</b>"),
+    ("[a=https://x.io]{ti[/a]tle}", '<a href="https://x.io">{title}</a>'),
+    ("[b]Ep[/b] {episode} [i]x[/i]", "<b>Ep</b> {episode} <i>x</i>"),  # a format that does not touch a variable is left alone
+    ("{[b]nonsense[/b]}", "{<b>nonsense</b>}"),  # only the variables the caption knows are held together
+])
+def test_a_format_never_cuts_a_variable_in_two(markup, expected):
+    assert html_of(markup) == expected
+
+
+# ---------------------------------------------------------------------------
+# it always says the same as the message did
+# ---------------------------------------------------------------------------
+
+
+def _coverage(text, entities):
+    units = add_surrogate(text)
+    covered = [set() for _ in units]
+    for e in entities:
+        for i in range(max(e.offset, 0), min(e.offset + e.length, len(units))):
+            covered[i].add((type(e).__name__, getattr(e, "url", None)))
+    return covered
+
+
+def test_random_messages_read_back_exactly_the_way_they_were_formatted():
+    """Whatever mix of overlapping formats, emoji and characters that need escaping: Telethon's own parser (which sends the
+    caption) reads back the same text with every character formatted the way it was."""
+    rng = random.Random(11)
+    alphabet = list("abc def") + ["🔥", "&", "<", ">", "\n", "é", "👨‍👩‍👧", '"', "'"]
+    kinds = [types.MessageEntityBold, types.MessageEntityItalic, types.MessageEntityUnderline, types.MessageEntityStrike, types.MessageEntityCode]
+
+    def off_the_middle_of_an_emoji(units, at):
+        while at < len(units) and "\udc00" <= units[at] <= "\udfff":
+            at += 1
+        return at
+
+    checked = 0
+    for _ in range(800):
+        text = "".join(rng.choice(alphabet) for _ in range(rng.randint(1, 16)))
+        units = add_surrogate(text)
+        entities = []
+        for _ in range(rng.randint(1, 5)):
+            a = rng.randrange(len(units))
+            a, b = off_the_middle_of_an_emoji(units, a), off_the_middle_of_an_emoji(units, rng.randint(a + 1, len(units)))
+            if b > a:
+                entities.append(rng.choice(kinds)(a, b - a))
+        cuts = sorted(rng.sample(range(len(units) + 1), min(len(units) + 1, 4)))  # links that do not overlap one another
+        for n in range(0, len(cuts) - 1, 2):
+            a, b = off_the_middle_of_an_emoji(units, cuts[n]), off_the_middle_of_an_emoji(units, cuts[n + 1])
+            if b > a and rng.random() < 0.5:
+                entities.append(types.MessageEntityTextUrl(a, b - a, f'https://x.io/?n={n}&q="a"'))
+        stored = from_entities(text, entities)
+        if stored is None:
+            continue
+        checked += 1
+        lead = len(units) - len(units.lstrip())
+        moved = [type(e)(e.offset - lead, e.length, **({"url": e.url} if hasattr(e, "url") else {})) for e in entities]
+        read_text, read_entities = telegram_html.parse(stored)
+        assert read_text == text.strip(), stored
+        assert _coverage(read_text, read_entities) == _coverage(text.strip(), moved), (text, stored)
+    assert checked > 300
+
+
+# ---------------------------------------------------------------------------
+# showing a stored caption
+# ---------------------------------------------------------------------------
+
+
+def test_a_stored_caption_is_shown_with_its_formatting():
+    assert preview("<b>{filename}</b> · {size}", 60) == "<b>{filename}</b> · {size}"
+    stored = html_of("[b]Watch[/b] {title} [i]episode {episode}[/i] on [a=https://t.me/Anime_X]our channel[/a]")
+    assert preview(stored, 1000) == stored
+
+
+def test_typed_html_is_shown_safely_whatever_was_typed():
+    for typed in ("</b>stray <i>open", "<b>x</b></b><u>", "<i><b>a</i></b>", "<<<>>>", "<a href=>x</a>", "<b>"):
+        shown = preview(typed, 60)
+        assert telegram_html.parse(shown)[0] == (telegram_html.parse(typed)[0] or "—")  # the same words
+        assert preview(shown, 60) == shown  # and what is shown is already well-formed: showing it again changes nothing
+    assert preview("</b>stray <i>open", 60) == "stray open"
+
+
+def test_special_characters_are_escaped_when_shown():
+    assert preview("Tom &amp; Jerry &lt;3 &gt;", 60) == "Tom &amp; Jerry &lt;3 &gt;"
+
+
+def test_a_long_caption_is_cut_and_the_tags_still_balance():
+    shown = preview("<b>" + "x" * 100 + "</b>", 20)
+    assert shown == "<b>" + "x" * 19 + "</b>…"
+    assert preview("short", 20) == "short"
+
+
+def test_a_cut_never_splits_an_emoji():
+    shown = preview("🔥" * 30, 11)
+    assert shown == "🔥🔥🔥🔥🔥…"
+    assert del_surrogate(add_surrogate(shown)) == shown
+
+
+def test_an_empty_caption_is_a_dash():
+    assert preview("", 20) == "—"

tests/test_share_app.py

@@ -213,7 +213,7 @@ async def test_opening_a_shared_format_shows_it_and_offers_to_add_it(ctx):
 async def test_a_shared_format_shows_its_caption_and_the_copy_keeps_it(ctx):
     fmt, token = await share(ctx, caption="<i>{filename}</i> · {size}")
     ev = await say(ctx, ADMIN_CFG, f"/start fmt_{token}")
-    assert "<b>Caption</b> ▸ <code>&lt;i&gt;{filename}&lt;/i&gt; · {size}</code>" in ev.texts[0] and "· default" not in ev.texts[0]
+    assert "<b>Caption</b> ▸ <i>{filename}</i> · {size}" in ev.texts[0] and "· default" not in ev.texts[0]
     await click(ctx, ADMIN_CFG, f"fs:add:{token}")
     (mine,) = await ctx.db.list_formats(ADMIN_CFG)
     assert mine["caption"] == "<i>{filename}</i> · {size}"  # theirs from now on
@@ -222,7 +222,7 @@ async def test_a_shared_format_shows_its_caption_and_the_copy_keeps_it(ctx):
 
     _plain, plain_token = await share(ctx, "Dragon Ball")  # without a caption of its own: the file name in bold
     ev = await say(ctx, ADMIN_CFG, f"/start fmt_{plain_token}")
-    assert "<b>Caption</b> ▸ <code>&lt;b&gt;{filename}&lt;/b&gt;</code> · default" in ev.texts[0]
+    assert "<b>Caption</b> ▸ <b>{filename}</b> · default" in ev.texts[0]
 
 
 async def test_a_plain_start_still_opens_the_menu(ctx):

tests/tgfakes.py

@@ -20,6 +20,44 @@ from telethon.tl.custom import InlineBuilder
 from telethon.tl.types import MessageEntityCode
 
 
+_ENTITY_TAGS = {
+    "b": types.MessageEntityBold, "i": types.MessageEntityItalic, "u": types.MessageEntityUnderline,
+    "s": types.MessageEntityStrike, "code": types.MessageEntityCode, "quote": types.MessageEntityBlockquote,
+    "spoiler": types.MessageEntitySpoiler, "tag": types.MessageEntityHashtag, "at": types.MessageEntityMention,
+}
+
+
+def telegram_text(markup: str) -> Tuple[str, List[Any]]:
+    """A message the way Telegram delivers it: the plain text and the formatting as entities.
+
+    ``"/caption [b]Hi[/b] [a=https://x.io]a link[/a]"`` -> ``("/caption Hi a link", [command, bold, link])``.  Tags: b i u s code
+    quote spoiler tag at, ``[pre=python]``, ``[a=url]``; tags may overlap.  Offsets count UTF-16 units, like Telegram's.
+    """
+    text, entities, open_tags, position = "", [], [], 0
+    for m in re.finditer(r"\[(/?)(\w+)(?:=([^\]]*))?\]", markup):
+        text += markup[position:m.start()]
+        position = m.end()
+        closing, tag, argument = m.groups()
+        here = len(add_surrogate(text))
+        if not closing:
+            open_tags.append((tag, argument, here))
+            continue
+        index = max(i for i, t in enumerate(open_tags) if t[0] == tag)
+        _tag, argument, offset = open_tags.pop(index)
+        length = here - offset
+        if tag == "a":
+            entities.append(types.MessageEntityTextUrl(offset, length, argument))
+        elif tag == "pre":
+            entities.append(types.MessageEntityPre(offset, length, argument or ""))
+        else:
+            entities.append(_ENTITY_TAGS[tag](offset, length))
+    text += markup[position:]
+    command = re.match(r"/\w+(?:@\w+)?", text)
+    if command:  # Telegram marks the command of a message as an entity of its own
+        entities.insert(0, types.MessageEntityBotCommand(0, command.end()))
+    return text, sorted(entities, key=lambda e: e.offset)
+
+
 def make_file(name: Optional[str] = "Movie.mkv", size: int = 5 * 1024 * 1024, mime: str = "video/x-matroska",
               ext: Optional[str] = None, *, width: Optional[int] = None, height: Optional[int] = None,
               duration: Optional[float] = None) -> SimpleNamespace:
@@ -72,8 +110,9 @@ class FakeEvent:
     """A ``NewMessage`` event."""
 
     def __init__(self, sender: int, text: str = "", *, message: Optional[FakeMessage] = None, reply: Any = None,
-                 mid: int = 1, chat: Optional[int] = None, private: bool = True, photo_bytes: Optional[bytes] = None):
-        self.message = message if message is not None else FakeMessage(text, mid=mid, sender=sender)
+                 mid: int = 1, chat: Optional[int] = None, private: bool = True, photo_bytes: Optional[bytes] = None,
+                 entities: Any = None):
+        self.message = message if message is not None else FakeMessage(text, mid=mid, sender=sender, entities=entities)
         self.message.sender_id = sender
         self.sender_id = sender
         self.chat_id = sender if chat is None else chat
@@ -118,6 +157,7 @@ class FakeCallback:
         self.id = 9
         self.query = SimpleNamespace(data=self.data, chat_instance=1, msg_id=msg_id)
         self.edits: List[tuple] = []
+        self.edit_kwargs: List[dict] = []  # the other arguments of every edit(), e.g. link_preview
         self.answers: List[tuple] = []
         self.deleted = False
         self._photo = photo
@@ -126,6 +166,7 @@ class FakeCallback:
     async def edit(self, text: Optional[str] = None, **kw: Any) -> None:
         refuse_share_button(getattr(self, "client", None), kw.get("buttons"))
         self.edits.append((text, kw.get("buttons")))
+        self.edit_kwargs.append(kw)
 
     async def answer(self, message: Optional[str] = None, *, alert: bool = False, **kw: Any) -> None:
         self.answers.append((message, alert))