File size: 3,178 Bytes
e75778c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 | import re
WORD = re.compile(r'(?:[^\W_]|[̀-ͯ᪰-᷀-᷿⃐-︠-︯])+')
PARTICLES = {'van', 'von', 'der', 'den', 'de', 'la', 'le', 'du', 'da', 'dos', 'di', 'del', 'al', 'el', 'bin', 'bint', 'ibn', 'ap', 'ab', 'mac', 'st'}
JOIN = set("-'’‘`.·‐‑–")
def char_scores(text, offsets, probs):
cp = [0.0] * len(text)
for (a, b), p in zip(offsets, probs):
for i in range(a, b):
if p > cp[i]:
cp[i] = p
return cp
def decode(text, offsets, probs, tau):
n = len(text)
cp = char_scores(text, offsets, probs)
mask = bytearray(n)
for m in WORD.finditer(text):
a, b = m.span()
if max(cp[a:b]) >= tau:
mask[a:b] = b'\x01' * (b - a)
word = bytes(mask)
for i, ch in enumerate(text):
if word[i] or ch.isspace():
continue
if cp[i] >= tau:
mask[i] = 1
elif ch in JOIN and 0 < i < n - 1 and word[i - 1] and word[i + 1]:
mask[i] = 1
elif ch == '.' and i >= 1 and word[i - 1] and text[i - 1].isupper() and (i < 2 or not text[i - 2].isalnum()):
j = i + 1
while j < n and text[j] in ' \t':
j += 1
if j < n and word[j]:
mask[i] = 1
spans, i = [], 0
while i < n:
if not mask[i]:
i += 1
continue
j = i
while j < n and mask[j]:
j += 1
if any(word[k] for k in range(i, j)):
spans.append([i, j])
i = j
merged = []
for a, b in spans:
if merged and not text[merged[-1][1]:a].strip(' \t'):
merged[-1][1] = b
else:
merged.append([a, b])
return merged
def initials_of(name):
words = [w for w in re.findall(r"[^\W\d_]+", name) if w.casefold() not in PARTICLES]
return ''.join(w[0] for w in words).upper() if len(words) >= 2 else None
def established_initials(text, spans, cp, seed=.9):
from_names = {initials_of(text[a:b]) for a, b in spans if ' ' in text[a:b]} - {None}
return {text[a:b] for a, b in spans
if re.fullmatch(r'[A-Z]{2,4}', text[a:b]) and max(cp[a:b]) >= seed and text[a:b] in from_names}
def with_initials(text, spans, cp):
found = established_initials(text, spans, cp)
if not found:
return spans
covered = bytearray(len(text))
for a, b in spans:
covered[a:b] = b'\x01' * (b - a)
out = [list(x) for x in spans]
for tok in found:
for m in re.finditer(r'(?<![\w.&/\-])' + tok + r'(?![\w.&/\-])', text):
a, b = m.span()
if any(covered[a:b]):
continue
start = text.rfind('\n', 0, a) + 1
end = text.find('\n', b)
line = text[start:a] + text[b:end if end >= 0 else len(text)]
letters = [c for c in line if c.isalpha()]
if letters and all(c.isupper() for c in letters):
continue
out.append([a, b])
out.sort()
return out
def spans_for(text, offsets, probs, tau):
spans = decode(text, offsets, probs, tau)
return with_initials(text, spans, char_scores(text, offsets, probs))
|