Spaces:
Running
Running
SilverElixir commited on
Commit ·
e89951a
1
Parent(s): 0a88ab0
Bound table separator match length for pathological lines
Browse files- lumen_formatting.py +11 -3
- tests/test_lumen_formatting.py +13 -0
lumen_formatting.py
CHANGED
|
@@ -8,6 +8,14 @@ from __future__ import annotations
|
|
| 8 |
import re
|
| 9 |
|
| 10 |
_TABLE_SEP_RE = re.compile(r"^\s*\|?\s*:?-{2,}:?\s*(\|\s*:?-{2,}:?\s*)*\|?\s*$")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 11 |
|
| 12 |
def _split_table_cells(line: str) -> list[str]:
|
| 13 |
s = line.strip()
|
|
@@ -28,7 +36,7 @@ def _convert_markdown_tables_to_lists(text: str) -> str:
|
|
| 28 |
n = len(lines)
|
| 29 |
while i < n:
|
| 30 |
line = lines[i]
|
| 31 |
-
if "|" in line and i + 1 < n and "-" in lines[i + 1] and
|
| 32 |
header_cells = _split_table_cells(line)
|
| 33 |
if len(header_cells) >= 2:
|
| 34 |
data_rows = []
|
|
@@ -61,7 +69,7 @@ def _table_block_line_indexes(lines: list[str]) -> set[int]:
|
|
| 61 |
idx: set[int] = set()
|
| 62 |
i, n = 0, len(lines)
|
| 63 |
while i < n:
|
| 64 |
-
if "|" in lines[i] and i + 1 < n and "-" in lines[i + 1] and
|
| 65 |
if len(_split_table_cells(lines[i])) >= 2:
|
| 66 |
j = i + 2
|
| 67 |
data = 0
|
|
@@ -531,7 +539,7 @@ def _md_to_rich_html(text: str) -> str:
|
|
| 531 |
n = len(lines)
|
| 532 |
while i < n:
|
| 533 |
line = lines[i]
|
| 534 |
-
if "|" in line and i + 1 < n and "-" in lines[i + 1] and
|
| 535 |
header_cells = _split_table_cells(line)
|
| 536 |
if len(header_cells) >= 2:
|
| 537 |
data_rows: list[list[str]] = []
|
|
|
|
| 8 |
import re
|
| 9 |
|
| 10 |
_TABLE_SEP_RE = re.compile(r"^\s*\|?\s*:?-{2,}:?\s*(\|\s*:?-{2,}:?\s*)*\|?\s*$")
|
| 11 |
+
# Регекс квадратичен на длинных пробельных хвостах: строку режем до match,
|
| 12 |
+
# разделитель шире капа в живых сообщениях не встречается.
|
| 13 |
+
_TABLE_SEP_MAX_LEN = 500
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
def _is_table_separator(line: str) -> bool:
|
| 17 |
+
"""Строка-разделитель markdown-таблицы с bounded match."""
|
| 18 |
+
return _TABLE_SEP_RE.match(line[:_TABLE_SEP_MAX_LEN]) is not None
|
| 19 |
|
| 20 |
def _split_table_cells(line: str) -> list[str]:
|
| 21 |
s = line.strip()
|
|
|
|
| 36 |
n = len(lines)
|
| 37 |
while i < n:
|
| 38 |
line = lines[i]
|
| 39 |
+
if "|" in line and i + 1 < n and "-" in lines[i + 1] and _is_table_separator(lines[i + 1]):
|
| 40 |
header_cells = _split_table_cells(line)
|
| 41 |
if len(header_cells) >= 2:
|
| 42 |
data_rows = []
|
|
|
|
| 69 |
idx: set[int] = set()
|
| 70 |
i, n = 0, len(lines)
|
| 71 |
while i < n:
|
| 72 |
+
if "|" in lines[i] and i + 1 < n and "-" in lines[i + 1] and _is_table_separator(lines[i + 1]):
|
| 73 |
if len(_split_table_cells(lines[i])) >= 2:
|
| 74 |
j = i + 2
|
| 75 |
data = 0
|
|
|
|
| 539 |
n = len(lines)
|
| 540 |
while i < n:
|
| 541 |
line = lines[i]
|
| 542 |
+
if "|" in line and i + 1 < n and "-" in lines[i + 1] and _is_table_separator(lines[i + 1]):
|
| 543 |
header_cells = _split_table_cells(line)
|
| 544 |
if len(header_cells) >= 2:
|
| 545 |
data_rows: list[list[str]] = []
|
tests/test_lumen_formatting.py
CHANGED
|
@@ -16,6 +16,7 @@ Telegram/Gemini/OpenRouter/рантайм-состояния бота, поэт
|
|
| 16 |
"""
|
| 17 |
import lumen_formatting
|
| 18 |
import pytest
|
|
|
|
| 19 |
|
| 20 |
|
| 21 |
|
|
@@ -76,6 +77,18 @@ def test_md_to_html_no_html_injection_via_markdown_markers():
|
|
| 76 |
assert result.count("<code>") == result.count("</code>")
|
| 77 |
|
| 78 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 79 |
def test_md_to_html_normalizes_raw_html_bold_tag():
|
| 80 |
# Регрессия: модель иногда пишет литеральные <b>/<i> теги вместо markdown
|
| 81 |
# (несмотря на инструкцию в system_prompt.py) — раньше это экранировалось
|
|
|
|
| 16 |
"""
|
| 17 |
import lumen_formatting
|
| 18 |
import pytest
|
| 19 |
+
import time
|
| 20 |
|
| 21 |
|
| 22 |
|
|
|
|
| 77 |
assert result.count("<code>") == result.count("</code>")
|
| 78 |
|
| 79 |
|
| 80 |
+
def test_table_separator_match_is_time_bounded():
|
| 81 |
+
# Квадратичный регекс на пробельных хвостах зависал: строка режется до match.
|
| 82 |
+
line = "| a | b |\n" + " " * 12000 + "-"
|
| 83 |
+
started = time.monotonic()
|
| 84 |
+
lumen_formatting._convert_markdown_tables_to_lists(line)
|
| 85 |
+
assert time.monotonic() - started < 1.0
|
| 86 |
+
# Обычный разделитель по-прежнему детектится.
|
| 87 |
+
assert lumen_formatting._is_table_separator("|---|---|") is True
|
| 88 |
+
assert lumen_formatting._is_table_separator(" --- | --- ") is True
|
| 89 |
+
assert lumen_formatting._is_table_separator("просто текст") is False
|
| 90 |
+
|
| 91 |
+
|
| 92 |
def test_md_to_html_normalizes_raw_html_bold_tag():
|
| 93 |
# Регрессия: модель иногда пишет литеральные <b>/<i> теги вместо markdown
|
| 94 |
# (несмотря на инструкцию в system_prompt.py) — раньше это экранировалось
|