Spaces:
Running
Running
SilverElixir commited on
Commit ·
281f95c
1
Parent(s): c5a914b
Split glued bullet lists without spaces or with middot
Browse files- lumen_formatting.py +16 -5
- system_prompt.py +2 -2
- tests/test_lumen_formatting.py +27 -0
lumen_formatting.py
CHANGED
|
@@ -144,24 +144,35 @@ def _normalize_bullet_markers(text: str) -> str:
|
|
| 144 |
# 2+ разделителя (т.е. уже 3 пункта: прод 26.09.2026 показал, что ровно
|
| 145 |
# трёхпунктовые списки — самый частый случай, а с порогом 3 они не ловились).
|
| 146 |
# Короткие "чай • кофе" (1 разделитель) и короткие абзацы — обычная проза, не трогаем.
|
|
|
|
|
|
|
| 147 |
_INLINE_BULLETS_MIN_SEPS = 2
|
| 148 |
_INLINE_BULLETS_MIN_LEN = 200
|
|
|
|
|
|
|
| 149 |
|
| 150 |
def _split_inline_bullets(text: str) -> str:
|
| 151 |
lines = text.split("\n")
|
| 152 |
skip = _table_block_line_indexes(lines)
|
| 153 |
out = []
|
| 154 |
for idx, line in enumerate(lines):
|
|
|
|
| 155 |
if (
|
| 156 |
idx not in skip
|
| 157 |
and len(line) >= _INLINE_BULLETS_MIN_LEN
|
| 158 |
-
and
|
| 159 |
and not _is_structural_line(line)
|
| 160 |
):
|
| 161 |
-
|
| 162 |
-
|
| 163 |
-
|
| 164 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 165 |
else:
|
| 166 |
out.append(line)
|
| 167 |
return "\n".join(out)
|
|
|
|
| 144 |
# 2+ разделителя (т.е. уже 3 пункта: прод 26.09.2026 показал, что ровно
|
| 145 |
# трёхпунктовые списки — самый частый случай, а с порогом 3 они не ловились).
|
| 146 |
# Короткие "чай • кофе" (1 разделитель) и короткие абзацы — обычная проза, не трогаем.
|
| 147 |
+
# Прод 05.10.2026: модель склеила тот же список без пробелов вокруг "•" —
|
| 148 |
+
# сепаратор ловим regex'ом с необязательными пробелами, плюс "·" тем же путём.
|
| 149 |
_INLINE_BULLETS_MIN_SEPS = 2
|
| 150 |
_INLINE_BULLETS_MIN_LEN = 200
|
| 151 |
+
_INLINE_BULLET_SEP_RE = re.compile(r"\s*[•·]\s*")
|
| 152 |
+
_INLINE_BULLET_START_RE = re.compile(r"\s*[•·]")
|
| 153 |
|
| 154 |
def _split_inline_bullets(text: str) -> str:
|
| 155 |
lines = text.split("\n")
|
| 156 |
skip = _table_block_line_indexes(lines)
|
| 157 |
out = []
|
| 158 |
for idx, line in enumerate(lines):
|
| 159 |
+
chunks: list[str] = []
|
| 160 |
if (
|
| 161 |
idx not in skip
|
| 162 |
and len(line) >= _INLINE_BULLETS_MIN_LEN
|
| 163 |
+
and len(_INLINE_BULLET_SEP_RE.findall(line)) >= _INLINE_BULLETS_MIN_SEPS
|
| 164 |
and not _is_structural_line(line)
|
| 165 |
):
|
| 166 |
+
chunks = [c.strip() for c in _INLINE_BULLET_SEP_RE.split(line)]
|
| 167 |
+
chunks = [c for c in chunks if c]
|
| 168 |
+
if len(chunks) >= 2:
|
| 169 |
+
if _INLINE_BULLET_START_RE.match(line):
|
| 170 |
+
# Строка начинается с маркера — вводной фразы нет, все куски пункты.
|
| 171 |
+
out.extend("• " + chunk for chunk in chunks)
|
| 172 |
+
else:
|
| 173 |
+
# Первый кусок — вводная фраза ("Вот моменты:"), дальше — пункты.
|
| 174 |
+
out.append(chunks[0])
|
| 175 |
+
out.extend("• " + chunk for chunk in chunks[1:])
|
| 176 |
else:
|
| 177 |
out.append(line)
|
| 178 |
return "\n".join(out)
|
system_prompt.py
CHANGED
|
@@ -116,7 +116,7 @@ SYSTEM_PROMPT = (
|
|
| 116 |
"NEVER literal HTML tags (<b>, <i>, etc.) — conversion happens bot-side after your answer; raw tags show as garbage text. "
|
| 117 |
"Math in LaTeX: inline $...$ (dollars tight, no spaces: '$E=mc^2$'), block $$...$$ on a new line; prices ('$80 000') are plain text, not formulas. "
|
| 118 |
"Structure with markdown headings (##, ###). "
|
| 119 |
-
"Lists: one item per line starting '- '; numbered items each on their own line too. "
|
| 120 |
"Comparisons and structured data go into real markdown tables (header row, |---|---| separator, short cells; **bold** and `code` work inside). "
|
| 121 |
"Extended explanations and stories go as connected prose in paragraphs — a solid bullet list instead of an explanation looks like a templated AI answer. List only what was asked to enumerate or rank; several items inside text are a comma phrase ('X, Y and Z'), not line markers.\n\n"
|
| 122 |
"OBJECTIVITY AND EVENHANDEDNESS:\n"
|
|
@@ -157,5 +157,5 @@ SYSTEM_PROMPT = (
|
|
| 157 |
"Joking criticism ('you are dumb'): no corporate defensiveness ('my goal is to be useful') — ask what is wrong, calmly, with light self-irony. "
|
| 158 |
"Polite refusals: no bureaucracy ('my task is to be safe', 'against my principles') — say directly what you will not do and offer real help instead. "
|
| 159 |
"Be ready to politely disagree and defend facts — see HONESTY.\n"
|
| 160 |
-
"EMOJI USAGE RULE: by default emojis are forbidden — no smileys, icons or decorative symbols, including list and heading decoration (no 🔍 ✅ ❓ 📌 📜 markers — lists only with
|
| 161 |
)
|
|
|
|
| 116 |
"NEVER literal HTML tags (<b>, <i>, etc.) — conversion happens bot-side after your answer; raw tags show as garbage text. "
|
| 117 |
"Math in LaTeX: inline $...$ (dollars tight, no spaces: '$E=mc^2$'), block $$...$$ on a new line; prices ('$80 000') are plain text, not formulas. "
|
| 118 |
"Structure with markdown headings (##, ###). "
|
| 119 |
+
"Lists: one item per line starting '- ', never join several bullets into a single paragraph; numbered items each on their own line too. "
|
| 120 |
"Comparisons and structured data go into real markdown tables (header row, |---|---| separator, short cells; **bold** and `code` work inside). "
|
| 121 |
"Extended explanations and stories go as connected prose in paragraphs — a solid bullet list instead of an explanation looks like a templated AI answer. List only what was asked to enumerate or rank; several items inside text are a comma phrase ('X, Y and Z'), not line markers.\n\n"
|
| 122 |
"OBJECTIVITY AND EVENHANDEDNESS:\n"
|
|
|
|
| 157 |
"Joking criticism ('you are dumb'): no corporate defensiveness ('my goal is to be useful') — ask what is wrong, calmly, with light self-irony. "
|
| 158 |
"Polite refusals: no bureaucracy ('my task is to be safe', 'against my principles') — say directly what you will not do and offer real help instead. "
|
| 159 |
"Be ready to politely disagree and defend facts — see HONESTY.\n"
|
| 160 |
+
"EMOJI USAGE RULE: by default emojis are forbidden — no smileys, icons or decorative symbols, including list and heading decoration (no 🔍 ✅ ❓ 📌 📜 markers — lists only with '- ' markers). Exception — only if the user explicitly asks for emojis, sets that style, or actively uses them in every message: then mirror restrainedly, 1-2 per answer. A wish to 'liven up' an answer is no reason to break the ban."
|
| 161 |
)
|
tests/test_lumen_formatting.py
CHANGED
|
@@ -297,6 +297,33 @@ def test_split_inline_bullets_splits_three_item_prod_answer():
|
|
| 297 |
assert all(line.startswith("• ") for line in lines)
|
| 298 |
|
| 299 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 300 |
def test_md_to_html_splits_inline_bullets_end_to_end():
|
| 301 |
items = ["тезис номер %d с подробным раскрытием мысли" % i for i in range(4)]
|
| 302 |
text = "Итоги сравнения моделей: " + " • ".join(items)
|
|
|
|
| 297 |
assert all(line.startswith("• ") for line in lines)
|
| 298 |
|
| 299 |
|
| 300 |
+
def test_split_inline_bullets_splits_glued_without_spaces():
|
| 301 |
+
# Прод 05.10.2026: скриншот — модель склеила список способностей вообще без
|
| 302 |
+
# пробелов вокруг "•", старая сетка на " • " его не видела.
|
| 303 |
+
text = (
|
| 304 |
+
"• Отвечать на вопросы, вести беседу.• Выполнять веб-поиск и использовать полученные данные. "
|
| 305 |
+
"•Читать открытые веб-страницы и анализировать их содержание.• Анализировать присланные фото, видео и документы."
|
| 306 |
+
)
|
| 307 |
+
assert len(text) >= 200
|
| 308 |
+
lines = lumen_formatting._split_inline_bullets(text).split("\n")
|
| 309 |
+
assert len(lines) == 4
|
| 310 |
+
assert all(line.startswith("• ") for line in lines)
|
| 311 |
+
assert "" not in lines
|
| 312 |
+
|
| 313 |
+
|
| 314 |
+
def test_split_inline_bullets_splits_middot_variant():
|
| 315 |
+
# Та же склейка маркером "·": другой символ, та же каша для читателя.
|
| 316 |
+
text = (
|
| 317 |
+
"Итоги такие: первый пункт с подробностями для длины строки и смыслом · второй пункт с подробностями "
|
| 318 |
+
"для длины строки и смыслом · третий пункт с подробностями для длины строки и смыслом · четвёртый пункт."
|
| 319 |
+
)
|
| 320 |
+
assert len(text) >= 200
|
| 321 |
+
lines = lumen_formatting._split_inline_bullets(text).split("\n")
|
| 322 |
+
assert len(lines) == 4
|
| 323 |
+
assert lines[0].startswith("Итоги такие: ")
|
| 324 |
+
assert all(line.startswith("• ") for line in lines[1:])
|
| 325 |
+
|
| 326 |
+
|
| 327 |
def test_md_to_html_splits_inline_bullets_end_to_end():
|
| 328 |
items = ["тезис номер %d с подробным раскрытием мысли" % i for i in range(4)]
|
| 329 |
text = "Итоги сравнения моделей: " + " • ".join(items)
|