File size: 608 Bytes
ed46c5d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 | """Cut degenerate repetition from generated transliterations: stop at the third consecutive
repeat of a word n-gram (n = 1..4) or of a character block of 2..12 characters."""
import re
def cut_loops(text):
words = text.split()
out = []
for w in words:
out.append(w)
for n in (1, 2, 3, 4):
if len(out) >= 3 * n and out[-n:] == out[-2 * n:-n] == out[-3 * n:-2 * n]:
del out[-2 * n:]
return " ".join(out)
s = " ".join(out)
m = re.search(r"(.{2,12}?)\1{2,}", s)
if m:
s = s[: m.start() + len(m.group(1))]
return s
|