File size: 4,862 Bytes
7ac8503 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 | import re
import decode as D
TITLE_BEFORE = re.compile(r'(?:\b(?:Mr|Mrs|Ms|Miss|Mx|Dr|Prof|Professor|Sir|Dame|Lord|Lady|Cllr|Councillor)\.?\s+)$')
TITLE = re.compile(r'^(?:(?:mr|mrs|ms|miss|mx|dr|prof|professor|sir|dame|lord|lady|cllr|councillor)\.?\s+)+', re.I)
PARTICLES = {'van', 'von', 'der', 'den', 'de', 'la', 'le', 'du', 'da', 'dos', 'di', 'del', 'al', 'el', 'bin', 'bint', 'ibn', 'ap', 'mac', 'st'}
ORG_TYPES = set('''council borough district county city metropolitan royal unitary parish town combined authority nhs foundation trust trusts
university hospital hospitals college school academy police fire rescue service services constabulary commissioner board agency department
ministry office health care integrated partnership housing transport commission ombudsman regulator executive committee institute library
libraries museum museums group association'''.split())
WORD = r"[^\W\d_]+(?:[’'\-][^\W\d_]+)*"
def flexible(name):
return r'\s+'.join(map(re.escape, name.split()))
def authority_ranges(text, authority):
if not authority or len(authority.split()) < 2:
return []
out = [(m.start(), m.end()) for m in re.finditer(r'(?<!\w)' + flexible(authority) + r'(?!\w)', text, re.I)]
words = re.findall(WORD, authority)
lead = []
for w in words:
if w.casefold() in ORG_TYPES:
if lead:
break
continue
lead.append(w)
if lead and len(lead) < len(words):
pat = rf'(?<!\w){flexible(" ".join(lead))}(?:[ \t]+{WORD}){{0,2}}[ \t]+(?:{"|".join(sorted(ORG_TYPES))})(?!\w)'
for m in re.finditer(pat, text, re.I):
tail = re.findall(WORD, m.group(0))[len(lead):]
if tail and all(w.casefold() in ORG_TYPES for w in tail):
out.append((m.start(), m.end()))
return out
def subtract(text, spans, ranges):
if not ranges:
return spans
out = []
for a, b in spans:
pieces = [(a, b)]
for x, y in ranges:
nxt = []
for p, q in pieces:
if y <= p or q <= x:
nxt.append((p, q))
continue
if p < x:
nxt.append((p, x))
if y < q:
nxt.append((y, q))
pieces = nxt
for p, q in pieces:
while p < q and text[p].isspace():
p += 1
while q > p and text[q - 1].isspace():
q -= 1
if p < q:
out.append([p, q])
return out
def requester_parts(name):
name = TITLE.sub('', name.strip()).strip()
parts = re.findall(WORD, name)
if len(parts) < 2:
return name, [], None
k = len(parts) - 1
while k > 1 and parts[k - 1].casefold() in PARTICLES:
k -= 1
variants = {parts[0], parts[-1]}
if k < len(parts) - 1:
variants.add(' '.join(parts[k:]))
variants = [v for v in variants if len(v) >= 3 and v.casefold() not in PARTICLES]
initials = ''.join(p[0] for p in parts if p.casefold() not in PARTICLES).upper()
return name, variants, initials if 2 <= len(initials) <= 4 else None
def merge(spans):
out = []
for a, b in sorted(map(list, spans)):
if out and a <= out[-1][1]:
out[-1][1] = max(out[-1][1], b)
else:
out.append([a, b])
return out
def apply(text, spans, rec, requester=None, authority=None, tau=.5, gate=.1):
protect = authority_ranges(text, authority)
spans = subtract(text, spans, protect)
if not requester or not requester.strip():
return merge(spans)
scores = None
def support(a, b):
nonlocal scores
if scores is None:
scores = D.char_scores(text, rec['offsets'], rec['probs'])
return max(scores[a:b], default=0.0)
name, variants, initials = requester_parts(requester)
add = []
if name and len(name.split()) >= 2:
add += [[m.start(), m.end()] for m in re.finditer(r'(?<!\w)' + flexible(name) + r'(?!\w)', text, re.I)]
elif name:
add += [[m.start(), m.end()] for m in re.finditer(r'(?<!\w)' + re.escape(name) + r'(?!\w)', text, re.I)
if support(m.start(), m.end()) >= gate]
for v in variants:
for m in re.finditer(r"(?<![\w\-’'])" + flexible(v) + r'(?![\w\-])', text):
a, b = m.span()
if any(x <= a and b <= y for x, y in protect):
continue
if TITLE_BEFORE.search(text[max(0, a - 16):a]) or support(a, b) >= gate:
add.append([a, b])
if initials:
for m in re.finditer(r'(?<![\w.])' + re.escape(initials) + r'(?![\w.])', text):
a, b = m.span()
if not any(x <= a and b <= y for x, y in protect) and support(a, b) >= gate:
add.append([a, b])
return merge(spans + add)
|