| """Cut degenerate repetition from generated transliterations: stop at the third consecutive | |
| repeat of a word n-gram (n = 1..4) or of a character block of 2..12 characters.""" | |
| import re | |
| def cut_loops(text): | |
| words = text.split() | |
| out = [] | |
| for w in words: | |
| out.append(w) | |
| for n in (1, 2, 3, 4): | |
| if len(out) >= 3 * n and out[-n:] == out[-2 * n:-n] == out[-3 * n:-2 * n]: | |
| del out[-2 * n:] | |
| return " ".join(out) | |
| s = " ".join(out) | |
| m = re.search(r"(.{2,12}?)\1{2,}", s) | |
| if m: | |
| s = s[: m.start() + len(m.group(1))] | |
| return s | |