Automated sync from GitHub

.gitattributes

@@ -1,8 +1,8 @@
-*.mp4 filter=lfs diff=lfs merge=lfs -text
-*.mp3 filter=lfs diff=lfs merge=lfs -text
 *.jpg filter=lfs diff=lfs merge=lfs -text
 *.jpeg filter=lfs diff=lfs merge=lfs -text
 *.png filter=lfs diff=lfs merge=lfs -text
 *.ttf filter=lfs diff=lfs merge=lfs -text
 *.otf filter=lfs diff=lfs merge=lfs -text
 *.webp filter=lfs diff=lfs merge=lfs -text
+*.mp4 filter=lfs diff=lfs merge=lfs -text
+*.mp3 filter=lfs diff=lfs merge=lfs -text

bot/db.py

@@ -73,7 +73,9 @@ class Admin(Base):
 
 
 class Format(Base):
-    __tablename__ = "formats"
+    # Not just "formats": the Postgres may be shared with another app that already has a table of that name, and
+    # create_all() leaves an existing table alone, so ours would never be created.
+    __tablename__ = "rename_formats"
 
     id: Mapped[int] = mapped_column(Integer, primary_key=True)
     user_id: Mapped[int] = mapped_column(BigInteger, index=True)

bot/errorlog.py

@@ -14,6 +14,7 @@ import re
 import time
 from collections import deque
 from dataclasses import dataclass
+from itertools import groupby
 from typing import Any, Awaitable, Callable, Deque, Dict, Iterable, List, Optional, Set, Tuple
 from urllib.parse import urlsplit
 
@@ -56,7 +57,7 @@ def describe_update(event: Any) -> str:
 
 
 _SQL_PARAMS = re.compile(r"\[parameters: [^\n]*\]")  # SQLAlchemy prints the values it bound, e.g. a saved session
-_TOKENISH = re.compile(r"[A-Za-z0-9_\-+/=]{60,}")  # session strings, access tokens, long hashes
+_TOKENISH = re.compile(r"[A-Za-z0-9_\-+=]{100,}")  # a session string is ~350 characters; no path or file name has 100 in a row
 
 
 def _mask_token(match: "re.Match[str]") -> str:
@@ -65,7 +66,7 @@ def _mask_token(match: "re.Match[str]") -> str:
 
 
 def redact(text: str, secrets: Iterable[str] = ()) -> str:
-    """Hide the configured secrets and anything that looks like a session string or access token."""
+    """Hide the configured secrets and anything that looks like a session string."""
     for secret in secrets:
         if len(secret) >= 6:
             text = text.replace(secret, "***")
@@ -101,14 +102,63 @@ def _tail(text: str, units: int) -> str:
     return tail
 
 
+def _head(text: str, units: int) -> str:
+    """The beginning of ``text`` that fits in ``units`` UTF-16 units, ending at the end of a line."""
+    used, end = 0, len(text)
+    for i, char in enumerate(text):
+        used += 2 if ord(char) > 0xFFFF else 1
+        if used > units:
+            end = i
+            break
+    if end == len(text):
+        return text
+    head = text[:end]
+    newline = head.rfind("\n")
+    return head[:newline] if newline != -1 else head
+
+
+_LIBRARY_FRAME = re.compile(r'  File ".*[\\/](?:site-packages|dist-packages|lib[\\/]python[\d.]+)[\\/]')
+_CUT = "\n…\n"
+
+
+def _elide(trace: str, room: int) -> str:
+    """Make a traceback fit ``room`` by replacing runs of frames from libraries (SQLAlchemy, Telethon, asyncio...) with
+    one line each: the bot's own frames and the error stay.  Runs of three or more go first, then every run."""
+    if _u16(trace) <= room:
+        return trace
+    blocks: List[List[str]] = []  # a frame is its "File" line and the source lines under it; any other line is a block of its own
+    for line in trace.split("\n"):
+        if blocks and line.startswith("    ") and blocks[-1][0].startswith("  File "):
+            blocks[-1].append(line)
+        else:
+            blocks.append([line])
+    text = trace
+    for shortest_run in (3, 1):
+        lines: List[str] = []
+        for library, group in groupby(blocks, key=lambda block: bool(_LIBRARY_FRAME.match(block[0]))):
+            frames = list(group)
+            if library and len(frames) >= shortest_run:
+                lines.append(f"  [… {len(frames)} library frame{'s' if len(frames) != 1 else ''} …]")
+            else:
+                lines.extend(line for frame in frames for line in frame)
+        text = "\n".join(lines)
+        if _u16(text) <= room:
+            break
+    return text
+
+
 def _fit(head: str, trace: str) -> str:
-    """``head`` + the traceback in at most ``MAX_TEXT``: when it is too long the end is kept, as that holds the error."""
+    """``head`` + the traceback in at most ``MAX_TEXT``.  What does not fit goes in this order: frames from libraries,
+    then the middle of the traceback: its first frames (where it came from) and its end (the error) are what matter."""
     if not trace:
         return clip(head, MAX_TEXT // 2)
     head = clip(head, HEAD_LIMIT)
     room = MAX_TEXT - _u16(head) - 2
+    trace = _elide(trace, room)
     if _u16(trace) > room:
-        trace = "…\n" + _tail(trace, room - 2)
+        room -= len(_CUT)
+        first = room * 2 // 5
+        trace = _head(trace, first) + _CUT + _tail(trace, room - first)
     return head + "\n\n" + trace
 
 

tests/test_db.py

@@ -76,6 +76,33 @@ async def test_admins_and_kv(db):
     assert await db.kv_get("session") is None
 
 
+async def test_a_foreign_table_called_formats_in_a_shared_database_is_left_alone(tmp_path):
+    """The Postgres can be shared with another app whose "formats" table has other columns (that broke /rename once)."""
+    import sqlite3
+
+    path = tmp_path / "shared.db"
+    other_app = sqlite3.connect(path)
+    other_app.execute("CREATE TABLE formats (id INTEGER PRIMARY KEY, name TEXT, body TEXT)")
+    other_app.execute("INSERT INTO formats (name, body) VALUES ('their format', 'their body')")
+    other_app.commit()
+    other_app.close()
+
+    database = Database(f"sqlite+aiosqlite:///{path}")
+    await database.init()
+    try:
+        created = await database.create_format(1, "One Piece", "{title} E{episode}")
+        assert [f["name"] for f in await database.list_formats(1)] == ["One Piece"]
+        assert (await database.get_format(1, created["id"]))["template"] == "{title} E{episode}"
+        assert await database.delete_format(1, created["id"]) is True
+    finally:
+        await database.close()
+
+    other_app = sqlite3.connect(path)
+    assert other_app.execute("SELECT name, body FROM formats").fetchall() == [("their format", "their body")]
+    assert [r[1] for r in other_app.execute("PRAGMA table_info(formats)")] == ["id", "name", "body"]
+    other_app.close()
+
+
 async def test_first_use_by_many_concurrent_handlers_does_not_collide(db):
     """A forwarded batch is handled by several tasks at once; all of them look the new user up together."""
     import asyncio

tests/test_errorlog.py

@@ -123,18 +123,18 @@ def test_configured_secrets_are_hidden_but_short_values_are_left_alone():
     assert redact("token abc123XYZ789 and abc", ["abc123XYZ789", "abc"]) == "token *** and abc"
 
 
-def test_session_strings_and_access_tokens_are_hidden():
+def test_session_strings_are_hidden():
     session = "1" + "A1b2C3_-" * 40
-    telegraph = "b968da509bb76866c35425099bc0989a5ec3b32997d55286c657e6994bbb"
-    text = f"bound {session} and {telegraph} here"
-    assert redact(text) == "bound *** and *** here"
+    assert redact(f"bound {session} here") == "bound *** here"
 
 
 def test_ordinary_text_survives():
     for text in (
         "File \"/home/user/app/bot/app.py\", line 1021, in _mediainfo",
+        "File \"/tmp/claude-0/-home-claude/77cec5a8-65b1-5e4a-87df-15495b60ec66/scratchpad/venv312/lib/python3.12/site-packages/x.py\"",
         "1147 - One Piece [Dual] 720p @Anime_X_Universe.mkv → One Piece - E1147.mkv (276.61 MB)",
-        "-" * 80,
+        "Naruto_Shippuden_Episode_123_Dual_Audio_English_Japanese_Subbed_1080p_HEVC_x265_10bit.mkv",
+        "-" * 150,
     ):
         assert redact(text) == text
 
@@ -198,12 +198,71 @@ def test_secrets_are_hidden_in_the_message_and_in_the_traceback():
     assert "TOP-SECRET-VALUE" not in body and body.count("***") >= 2
 
 
-def test_a_long_traceback_keeps_its_end_and_fits_in_one_message():
+def test_a_long_traceback_keeps_its_first_frames_and_its_end_and_fits_in_one_message():
     _signature, body = build_report(make_record("Error handling update:", very_deep))
     text = visible(body)
     assert errorlog._u16(text) <= errorlog.MAX_TEXT
-    assert text.startswith("Error handling update:\n\n…\n  File ")  # cut at the beginning of a line
-    assert text.endswith("ValueError: bottom")
+    first, last = text.split("\n…\n")  # only the middle is cut
+    assert first.startswith("Error handling update:\n\nTraceback (most recent call last):\n  File ")
+    assert "in make_record" in first and "in very_deep" in first  # where it came from
+    assert re.search(r", in f\d+$", first)  # cut after a whole line
+    assert last.startswith("  File ") and last.endswith("ValueError: bottom")  # from the start of a line, up to the error
+
+
+def test_a_traceback_that_fits_is_left_whole():
+    text = visible(build_report(make_record("Error handling update:", chained))[1])
+    assert "…" not in text and "in make_record" in text
+
+
+def library_failure(depth=40):
+    """Fail ``depth`` calls deep inside code that looks like it lives in site-packages, the way SQLAlchemy's frames do."""
+    scope = {}
+    source = "\n".join(f"def g{i}():\n    return g{i + 1}()" for i in range(depth)) + f"\ndef g{depth}():\n    raise ValueError('inside the library')"
+    exec(compile(source, "/home/user/.local/lib/python3.12/site-packages/somelib/core.py", "exec"), scope)  # noqa: S102
+    scope["g0"]()
+
+
+def test_library_frames_are_collapsed_before_anything_of_the_bots_own_is_cut():
+    text = visible(build_report(make_record("Error handling update:", library_failure))[1])
+    assert errorlog._u16(text) <= errorlog.MAX_TEXT
+    assert "in make_record" in text and "in library_failure" in text  # the bot's own frames stay
+    assert "[… 41 library frames …]" in text and "somelib" not in text
+    assert "\n…\n" not in text  # nothing else had to go
+    assert text.endswith("ValueError: inside the library")
+
+
+def test_only_as_much_of_the_library_frames_is_collapsed_as_needed():
+    def frame(path, n):
+        return f'  File "{path}", line {n}, in f{n}\n    call{n}()'
+
+    lib, mine = "/usr/local/lib/python3.12/site-packages/x.py", "/home/user/app/bot/app.py"
+    trace = "\n".join(
+        ["Traceback (most recent call last):", frame(mine, 1)] + [frame(lib, n) for n in range(10, 15)]
+        + [frame(mine, 2), frame(lib, 30), frame(mine, 3), "ValueError: boom"]
+    )
+    assert errorlog._elide(trace, 10_000) == trace  # fits: left alone
+    some = errorlog._elide(trace, len(trace) - 50)  # the run of five goes, the single library frame stays
+    assert "[… 5 library frames …]" in some and 'x.py", line 30' in some
+    most = errorlog._elide(trace, 150)  # every run goes
+    assert "[… 1 library frame …]" in most and 'x.py' not in most
+    assert most.endswith("ValueError: boom") and 'app.py", line 1' in most
+
+
+def test_a_chained_traceback_keeps_each_exception_line_when_its_library_frames_are_collapsed():
+    def frame(path, n):
+        return f'  File "{path}", line {n}, in f{n}\n    call{n}()'
+
+    lib = "/usr/local/lib/python3.12/site-packages/asyncpg/protocol.py"
+    mine = "/home/user/app/bot/db.py"
+    trace = "\n".join(
+        ["Traceback (most recent call last):"] + [frame(lib, n) for n in range(5)] + ["asyncpg.UndefinedColumnError: no such column", "",
+         "The above exception was the direct cause of the following exception:", "", "Traceback (most recent call last):", frame(mine, 7)]
+        + [frame(lib, n) for n in range(20, 30)] + ["sqlalchemy.exc.ProgrammingError: column formats.data does not exist"]
+    )
+    text = errorlog._elide(trace, 300)
+    assert text.count("library frames …]") == 2
+    assert "asyncpg.UndefinedColumnError: no such column" in text and "direct cause of the following exception" in text
+    assert 'db.py", line 7' in text and text.endswith("column formats.data does not exist")
 
 
 def test_emoji_count_double_towards_the_limit():