Upload pasita.py with huggingface_hub
Browse files
pasita.py
CHANGED
|
@@ -305,6 +305,10 @@ def _fix_double_heading(text: str) -> str:
|
|
| 305 |
return "\n".join(out)
|
| 306 |
|
| 307 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 308 |
def _cut_repeated_tail_paragraphs(text: str, input_text: str) -> str:
|
| 309 |
paras = re.split(r"\n\s*\n", text.strip())
|
| 310 |
if len(paras) < 2:
|
|
@@ -511,6 +515,7 @@ def postprocess(raw: str, input_text: str | None = None) -> str:
|
|
| 511 |
text = _dedup_leading_h1(text)
|
| 512 |
text = _repair_headings(text, input_text or "")
|
| 513 |
text = _fix_double_heading(text)
|
|
|
|
| 514 |
text = _cut_after_completion(text, input_text or "")
|
| 515 |
text = _cut_repeated_tail_paragraphs(text, input_text or "")
|
| 516 |
text = _cut_repeated_line_blocks(text)
|
|
|
|
| 305 |
return "\n".join(out)
|
| 306 |
|
| 307 |
|
| 308 |
+
def _space_heading_markers(text: str) -> str:
|
| 309 |
+
return re.sub(r"(?m)^(\s*#{1,6})(?=[^\s#])", r"\1 ", text)
|
| 310 |
+
|
| 311 |
+
|
| 312 |
def _cut_repeated_tail_paragraphs(text: str, input_text: str) -> str:
|
| 313 |
paras = re.split(r"\n\s*\n", text.strip())
|
| 314 |
if len(paras) < 2:
|
|
|
|
| 515 |
text = _dedup_leading_h1(text)
|
| 516 |
text = _repair_headings(text, input_text or "")
|
| 517 |
text = _fix_double_heading(text)
|
| 518 |
+
text = _space_heading_markers(text)
|
| 519 |
text = _cut_after_completion(text, input_text or "")
|
| 520 |
text = _cut_repeated_tail_paragraphs(text, input_text or "")
|
| 521 |
text = _cut_repeated_line_blocks(text)
|