"""Finds vocabulary phrases, tenant names and literal values (domains, numbers ...) in the tokens. Output is one Item per position: a matched phrase (with every sense it can have), a literal, an unknown word or a punctuation mark. Deciding which sense is meant is interpret.py's job. """ from __future__ import annotations import re from dataclasses import dataclass, field from .lexicon import Lexicon, Sense, load from .platform.context import PlatformContext from .text import Token, stem, is_serial FUZZY_MIN = 5 EXT_ID = re.compile(r'^[a-p]{32}$') # a Chrome / Edge extension id @dataclass class Item: i: int # token range [i, j) j: int start: int # character range in the normalised text end: int text: str senses: list[Sense] = field(default_factory=list) method: str = 'none' # exact | stem | fuzzy | context | literal | none | punct score: float = 1.0 typed: str = '' # what the user actually typed, when it was corrected (fuzzy) def has(self, *slots: str) -> bool: return any(s.slot in slots for s in self.senses) def get(self, slot: str) -> Sense | None: return next((s for s in self.senses if s.slot == slot), None) def values(self, slot: str) -> list: return [s.value for s in self.senses if s.slot == slot] @property def is_punct(self) -> bool: return self.method == 'punct' def _osa(a: str, b: str, limit: int) -> int: """Optimal string alignment distance (Levenshtein + adjacent transposition), early exit.""" if abs(len(a) - len(b)) > limit: return limit + 1 prev2: list[int] | None = None prev = list(range(len(b) + 1)) for i in range(1, len(a) + 1): cur = [i] + [0] * len(b) best = cur[0] for j in range(1, len(b) + 1): cost = 0 if a[i - 1] == b[j - 1] else 1 v = min(prev[j] + 1, cur[j - 1] + 1, prev[j - 1] + cost) if prev2 is not None and i > 1 and j > 1 and a[i - 1] == b[j - 2] and a[i - 2] == b[j - 1]: v = min(v, prev2[j - 2] + 1) cur[j] = v best = min(best, v) if best > limit: return limit + 1 prev2, prev = prev, cur return prev[-1] class _Overlay: """Tenant names (groups, custom rules and lists, connectors) indexed like vocabulary.""" def __init__(self, ctx: PlatformContext, lx: Lexicon): from .text import tokenize self.exact: dict[tuple[str, ...], list[Sense]] = {} self.stemmed: dict[tuple[str, ...], list[Sense]] = {} self.fuzzy: dict[str, list[Sense]] = {} self.max_len = 1 def add(name: str, sense: Sense) -> None: toks = tuple(t.text for t in tokenize(name) if t.kind != 'punct' or t.text in '&/') if not toks: return for d, key in ((self.exact, toks), (self.stemmed, tuple(stem(t) for t in toks))): lst = d.setdefault(key, []) if sense not in lst: lst.append(sense) self.max_len = max(self.max_len, len(toks)) if len(toks) == 1 and len(toks[0]) >= FUZZY_MIN and toks[0].isalpha() and toks[0] not in lx.no_fuzzy: self.fuzzy.setdefault(toks[0], []).append(sense) for g in ctx.groups: sense = Sense('group', g.id, g.name) add(g.name, sense) # "Finance Team" is also "finance"; "Sales-Ops" is also "sales ops". words = g.name.lower().replace('-', ' ').replace('_', ' ').split() if len(words) > 1 and words[-1] in ('team', 'group', 'dept', 'department', 'users', 'staff'): add(' '.join(words[:-1]), sense) if '-' in g.name or '_' in g.name: add(' '.join(words), sense) for r in ctx.rules: if not r.builtin: add(r.name, Sense('rule_ref', r.id, r.name)) for b in ctx.blockLists: if not b.builtin: add(b.name, Sense('blist', b.id, b.name)) for w in ctx.webCategories: add(w.name, Sense('wcat', w.name, w.name)) # the tenant's own category name, as stored for ch, conns in ctx.connectors.items(): for c in conns: add(c.name, Sense('connector', (ch, c.id), c.name)) # The tenant's regex detectors: "employee code" -> EMPLOYEE_CODE, by its name and by its label. for x in ctx.detectors: sense = Sense('entity', x.label, x.label) add(x.name, sense) add(x.label.replace('_', ' '), sense) def match(tokens: list[Token], ctx: PlatformContext, lx: Lexicon | None = None) -> list[Item]: lx = lx or load() ov = _Overlay(ctx, lx) tokens = _split_unknown_compounds(tokens, lx) texts = [t.text for t in tokens] items: list[Item] = [] i = 0 n = len(tokens) while i < n: tok = tokens[i] if tok.kind == 'punct' and tok.text not in '/&:': items.append(Item(i, i + 1, tok.start, tok.end, tok.text, method='punct')) i += 1 continue found = _longest(texts, i, lx, ov, tokens) if found: L, senses, method = found j = i + L items.append(Item(i, j, tok.start, tokens[j - 1].end, ' '.join(texts[i:j]), senses, method)) i = j continue if tok.kind == 'punct': items.append(Item(i, i + 1, tok.start, tok.end, tok.text, method='punct')) i += 1 continue lit = _literal(tok) if lit: items.append(Item(i, i + 1, tok.start, tok.end, tok.text, [lit], 'literal')) i += 1 continue fz = _fuzzy(tok, lx, ov) if fz: senses, target, score = fz items.append(Item(i, i + 1, tok.start, tok.end, target, senses, 'fuzzy', score, typed=tok.text)) i += 1 continue items.append(Item(i, i + 1, tok.start, tok.end, tok.text)) i += 1 return items def _longest(texts: list[str], i: int, lx: Lexicon, ov: _Overlay, tokens: list[Token]): n = len(texts) for L in range(min(max(lx.max_len, ov.max_len), n - i), 0, -1): # A phrase never spans a sentence break. if any(tokens[k].kind == 'punct' and tokens[k].text not in '/&:' for k in range(i, i + L)): continue key = tuple(texts[i:i + L]) senses = _tenant_first(ov.exact.get(key, []), lx.exact.get(key, [])) if senses: return L, _dedupe(senses), 'context' if key in ov.exact else 'exact' skey = tuple(stem(t) for t in key) senses = _tenant_first(ov.stemmed.get(skey, []), lx.stemmed.get(skey, [])) if senses: return L, _dedupe(senses), 'context' if skey in ov.stemmed else 'stem' return None def _tenant_first(tenant: list[Sense], vocab: list[Sense]) -> list[Sense]: """A tenant's own detector named exactly like a vocabulary phrase is what the admin means: "employee code" is their EMPLOYEE_CODE detector, not also the built-in EMPLOYEE_ID.""" if any(s.slot == 'entity' for s in tenant): vocab = [s for s in vocab if s.slot != 'entity'] return list(tenant) + list(vocab) def _dedupe(senses: list[Sense]) -> list[Sense]: out: list[Sense] = [] for s in senses: if s not in out: out.append(s) return out def _literal(tok: Token) -> Sense | None: if tok.kind == 'domain': return Sense('domain', tok.value) if tok.kind == 'email': return Sense('email_addr', tok.value) if tok.kind == 'number' and tok.value is not None: return Sense('number', tok.value) if tok.kind == 'time' and tok.value is not None: return Sense('time', tok.value) if tok.kind == 'quoted': return Sense('quoted', tok.value) if tok.kind == 'path': return Sense('path', tok.value) if tok.kind == 'word' and EXT_ID.match(tok.text): return Sense('ext_id', tok.text) if tok.kind == 'word' and is_serial(tok.text): return Sense('serial', tok.text.upper()) return None def _fuzzy(tok: Token, lx: Lexicon, ov: _Overlay): w = tok.text if tok.kind != 'word' or len(w) < FUZZY_MIN or not w.isalpha() or w in lx.no_fuzzy or w in lx.words: return None # Group names in another word form: "engineers" -> Engineering, "marketers" -> Marketing. forms = [] for target, senses in ov.fuzzy.items(): if not any(s.slot == 'group' for s in senses): continue cp = len(_common_prefix(w, target)) if cp >= 6 and cp >= 0.75 * min(len(w), len(target)): forms.append((target, senses)) if len(forms) == 1: target, senses = forms[0] return _dedupe(senses), target, 0.85 limit = 1 if len(w) <= 7 else 2 best: list[tuple[int, str, list[Sense]]] = [] for pool in (ov.fuzzy, lx.fuzzy_targets): for target, senses in pool.items(): if target[0] != w[0] or abs(len(target) - len(w)) > limit: continue d = _osa(w, target, limit) if d <= limit: best.append((d, target, senses)) if not best: return None best.sort(key=lambda x: x[0]) top = [b for b in best if b[0] == best[0][0]] # Two different words equally close: guessing would be a coin toss, so do not. if len({b[1] for b in top}) > 1: return None d, target, senses = top[0] return _dedupe(senses), target, round(1 - d / max(len(w), 1), 2) def _common_prefix(a: str, b: str) -> str: n = 0 while n < min(len(a), len(b)) and a[n] == b[n]: n += 1 return a[:n] def _split_unknown_compounds(tokens: list[Token], lx: Lexicon) -> list[Token]: """'pen-drive' is not a vocabulary word, but 'pen drive' is: split hyphenated unknowns.""" out: list[Token] = [] for t in tokens: if t.kind == 'word' and '-' in t.text and t.text not in lx.words and not is_serial(t.text): pos = t.start for part in t.text.split('-'): if part: out.append(Token(part, pos, pos + len(part), 'word')) pos += len(part) + 1 else: out.append(t) return out