| import re |
|
|
| import decode as D |
|
|
| TITLE_BEFORE = re.compile(r'(?:\b(?:Mr|Mrs|Ms|Miss|Mx|Dr|Prof|Professor|Sir|Dame|Lord|Lady|Cllr|Councillor)\.?\s+)$') |
| TITLE = re.compile(r'^(?:(?:mr|mrs|ms|miss|mx|dr|prof|professor|sir|dame|lord|lady|cllr|councillor)\.?\s+)+', re.I) |
| PARTICLES = {'van', 'von', 'der', 'den', 'de', 'la', 'le', 'du', 'da', 'dos', 'di', 'del', 'al', 'el', 'bin', 'bint', 'ibn', 'ap', 'mac', 'st'} |
| ORG_TYPES = set('''council borough district county city metropolitan royal unitary parish town combined authority nhs foundation trust trusts |
| university hospital hospitals college school academy police fire rescue service services constabulary commissioner board agency department |
| ministry office health care integrated partnership housing transport commission ombudsman regulator executive committee institute library |
| libraries museum museums group association'''.split()) |
| WORD = r"[^\W\d_]+(?:[’'\-][^\W\d_]+)*" |
|
|
|
|
| def flexible(name): |
| return r'\s+'.join(map(re.escape, name.split())) |
|
|
|
|
| def authority_ranges(text, authority): |
| if not authority or len(authority.split()) < 2: |
| return [] |
| out = [(m.start(), m.end()) for m in re.finditer(r'(?<!\w)' + flexible(authority) + r'(?!\w)', text, re.I)] |
| words = re.findall(WORD, authority) |
| lead = [] |
| for w in words: |
| if w.casefold() in ORG_TYPES: |
| if lead: |
| break |
| continue |
| lead.append(w) |
| if lead and len(lead) < len(words): |
| pat = rf'(?<!\w){flexible(" ".join(lead))}(?:[ \t]+{WORD}){{0,2}}[ \t]+(?:{"|".join(sorted(ORG_TYPES))})(?!\w)' |
| for m in re.finditer(pat, text, re.I): |
| tail = re.findall(WORD, m.group(0))[len(lead):] |
| if tail and all(w.casefold() in ORG_TYPES for w in tail): |
| out.append((m.start(), m.end())) |
| return out |
|
|
|
|
| def subtract(text, spans, ranges): |
| if not ranges: |
| return spans |
| out = [] |
| for a, b in spans: |
| pieces = [(a, b)] |
| for x, y in ranges: |
| nxt = [] |
| for p, q in pieces: |
| if y <= p or q <= x: |
| nxt.append((p, q)) |
| continue |
| if p < x: |
| nxt.append((p, x)) |
| if y < q: |
| nxt.append((y, q)) |
| pieces = nxt |
| for p, q in pieces: |
| while p < q and text[p].isspace(): |
| p += 1 |
| while q > p and text[q - 1].isspace(): |
| q -= 1 |
| if p < q: |
| out.append([p, q]) |
| return out |
|
|
|
|
| def requester_parts(name): |
| name = TITLE.sub('', name.strip()).strip() |
| parts = re.findall(WORD, name) |
| if len(parts) < 2: |
| return name, [], None |
| k = len(parts) - 1 |
| while k > 1 and parts[k - 1].casefold() in PARTICLES: |
| k -= 1 |
| variants = {parts[0], parts[-1]} |
| if k < len(parts) - 1: |
| variants.add(' '.join(parts[k:])) |
| variants = [v for v in variants if len(v) >= 3 and v.casefold() not in PARTICLES] |
| initials = ''.join(p[0] for p in parts if p.casefold() not in PARTICLES).upper() |
| return name, variants, initials if 2 <= len(initials) <= 4 else None |
|
|
|
|
| def merge(spans): |
| out = [] |
| for a, b in sorted(map(list, spans)): |
| if out and a <= out[-1][1]: |
| out[-1][1] = max(out[-1][1], b) |
| else: |
| out.append([a, b]) |
| return out |
|
|
|
|
| def apply(text, spans, rec, requester=None, authority=None, tau=.5, gate=.1): |
| protect = authority_ranges(text, authority) |
| spans = subtract(text, spans, protect) |
| if not requester or not requester.strip(): |
| return merge(spans) |
| scores = None |
|
|
| def support(a, b): |
| nonlocal scores |
| if scores is None: |
| scores = D.char_scores(text, rec['offsets'], rec['probs']) |
| return max(scores[a:b], default=0.0) |
|
|
| name, variants, initials = requester_parts(requester) |
| add = [] |
| if name and len(name.split()) >= 2: |
| add += [[m.start(), m.end()] for m in re.finditer(r'(?<!\w)' + flexible(name) + r'(?!\w)', text, re.I)] |
| elif name: |
| add += [[m.start(), m.end()] for m in re.finditer(r'(?<!\w)' + re.escape(name) + r'(?!\w)', text, re.I) |
| if support(m.start(), m.end()) >= gate] |
| for v in variants: |
| for m in re.finditer(r"(?<![\w\-’'])" + flexible(v) + r'(?![\w\-])', text): |
| a, b = m.span() |
| if any(x <= a and b <= y for x, y in protect): |
| continue |
| if TITLE_BEFORE.search(text[max(0, a - 16):a]) or support(a, b) >= gate: |
| add.append([a, b]) |
| if initials: |
| for m in re.finditer(r'(?<![\w.])' + re.escape(initials) + r'(?![\w.])', text): |
| a, b = m.span() |
| if not any(x <= a and b <= y for x, y in protect) and support(a, b) >= gate: |
| add.append([a, b]) |
| return merge(spans + add) |
|
|
