cloak-api / copilot /matcher.py
n1th1sh's picture
Upload 35 files
375880c verified
Raw History Blame Contribute Delete
10.3 kB
"""Finds vocabulary phrases, tenant names and literal values (domains, numbers ...) in the tokens.
Output is one Item per position: a matched phrase (with every sense it can have), a literal,
an unknown word or a punctuation mark. Deciding which sense is meant is interpret.py's job.
"""
from __future__ import annotations
import re
from dataclasses import dataclass, field
from .lexicon import Lexicon, Sense, load
from .platform.context import PlatformContext
from .text import Token, stem, is_serial
FUZZY_MIN = 5
EXT_ID = re.compile(r'^[a-p]{32}$') # a Chrome / Edge extension id
@dataclass
class Item:
i: int # token range [i, j)
j: int
start: int # character range in the normalised text
end: int
text: str
senses: list[Sense] = field(default_factory=list)
method: str = 'none' # exact | stem | fuzzy | context | literal | none | punct
score: float = 1.0
typed: str = '' # what the user actually typed, when it was corrected (fuzzy)
def has(self, *slots: str) -> bool:
return any(s.slot in slots for s in self.senses)
def get(self, slot: str) -> Sense | None:
return next((s for s in self.senses if s.slot == slot), None)
def values(self, slot: str) -> list:
return [s.value for s in self.senses if s.slot == slot]
@property
def is_punct(self) -> bool:
return self.method == 'punct'
def _osa(a: str, b: str, limit: int) -> int:
"""Optimal string alignment distance (Levenshtein + adjacent transposition), early exit."""
if abs(len(a) - len(b)) > limit:
return limit + 1
prev2: list[int] | None = None
prev = list(range(len(b) + 1))
for i in range(1, len(a) + 1):
cur = [i] + [0] * len(b)
best = cur[0]
for j in range(1, len(b) + 1):
cost = 0 if a[i - 1] == b[j - 1] else 1
v = min(prev[j] + 1, cur[j - 1] + 1, prev[j - 1] + cost)
if prev2 is not None and i > 1 and j > 1 and a[i - 1] == b[j - 2] and a[i - 2] == b[j - 1]:
v = min(v, prev2[j - 2] + 1)
cur[j] = v
best = min(best, v)
if best > limit:
return limit + 1
prev2, prev = prev, cur
return prev[-1]
class _Overlay:
"""Tenant names (groups, custom rules and lists, connectors) indexed like vocabulary."""
def __init__(self, ctx: PlatformContext, lx: Lexicon):
from .text import tokenize
self.exact: dict[tuple[str, ...], list[Sense]] = {}
self.stemmed: dict[tuple[str, ...], list[Sense]] = {}
self.fuzzy: dict[str, list[Sense]] = {}
self.max_len = 1
def add(name: str, sense: Sense) -> None:
toks = tuple(t.text for t in tokenize(name) if t.kind != 'punct' or t.text in '&/')
if not toks:
return
for d, key in ((self.exact, toks), (self.stemmed, tuple(stem(t) for t in toks))):
lst = d.setdefault(key, [])
if sense not in lst:
lst.append(sense)
self.max_len = max(self.max_len, len(toks))
if len(toks) == 1 and len(toks[0]) >= FUZZY_MIN and toks[0].isalpha() and toks[0] not in lx.no_fuzzy:
self.fuzzy.setdefault(toks[0], []).append(sense)
for g in ctx.groups:
sense = Sense('group', g.id, g.name)
add(g.name, sense)
# "Finance Team" is also "finance"; "Sales-Ops" is also "sales ops".
words = g.name.lower().replace('-', ' ').replace('_', ' ').split()
if len(words) > 1 and words[-1] in ('team', 'group', 'dept', 'department', 'users', 'staff'):
add(' '.join(words[:-1]), sense)
if '-' in g.name or '_' in g.name:
add(' '.join(words), sense)
for r in ctx.rules:
if not r.builtin:
add(r.name, Sense('rule_ref', r.id, r.name))
for b in ctx.blockLists:
if not b.builtin:
add(b.name, Sense('blist', b.id, b.name))
for w in ctx.webCategories:
add(w.name, Sense('wcat', w.name, w.name)) # the tenant's own category name, as stored
for ch, conns in ctx.connectors.items():
for c in conns:
add(c.name, Sense('connector', (ch, c.id), c.name))
# The tenant's regex detectors: "employee code" -> EMPLOYEE_CODE, by its name and by its label.
for x in ctx.detectors:
sense = Sense('entity', x.label, x.label)
add(x.name, sense)
add(x.label.replace('_', ' '), sense)
def match(tokens: list[Token], ctx: PlatformContext, lx: Lexicon | None = None) -> list[Item]:
lx = lx or load()
ov = _Overlay(ctx, lx)
tokens = _split_unknown_compounds(tokens, lx)
texts = [t.text for t in tokens]
items: list[Item] = []
i = 0
n = len(tokens)
while i < n:
tok = tokens[i]
if tok.kind == 'punct' and tok.text not in '/&:':
items.append(Item(i, i + 1, tok.start, tok.end, tok.text, method='punct'))
i += 1
continue
found = _longest(texts, i, lx, ov, tokens)
if found:
L, senses, method = found
j = i + L
items.append(Item(i, j, tok.start, tokens[j - 1].end, ' '.join(texts[i:j]), senses, method))
i = j
continue
if tok.kind == 'punct':
items.append(Item(i, i + 1, tok.start, tok.end, tok.text, method='punct'))
i += 1
continue
lit = _literal(tok)
if lit:
items.append(Item(i, i + 1, tok.start, tok.end, tok.text, [lit], 'literal'))
i += 1
continue
fz = _fuzzy(tok, lx, ov)
if fz:
senses, target, score = fz
items.append(Item(i, i + 1, tok.start, tok.end, target, senses, 'fuzzy', score, typed=tok.text))
i += 1
continue
items.append(Item(i, i + 1, tok.start, tok.end, tok.text))
i += 1
return items
def _longest(texts: list[str], i: int, lx: Lexicon, ov: _Overlay, tokens: list[Token]):
n = len(texts)
for L in range(min(max(lx.max_len, ov.max_len), n - i), 0, -1):
# A phrase never spans a sentence break.
if any(tokens[k].kind == 'punct' and tokens[k].text not in '/&:' for k in range(i, i + L)):
continue
key = tuple(texts[i:i + L])
senses = _tenant_first(ov.exact.get(key, []), lx.exact.get(key, []))
if senses:
return L, _dedupe(senses), 'context' if key in ov.exact else 'exact'
skey = tuple(stem(t) for t in key)
senses = _tenant_first(ov.stemmed.get(skey, []), lx.stemmed.get(skey, []))
if senses:
return L, _dedupe(senses), 'context' if skey in ov.stemmed else 'stem'
return None
def _tenant_first(tenant: list[Sense], vocab: list[Sense]) -> list[Sense]:
"""A tenant's own detector named exactly like a vocabulary phrase is what the admin means:
"employee code" is their EMPLOYEE_CODE detector, not also the built-in EMPLOYEE_ID."""
if any(s.slot == 'entity' for s in tenant):
vocab = [s for s in vocab if s.slot != 'entity']
return list(tenant) + list(vocab)
def _dedupe(senses: list[Sense]) -> list[Sense]:
out: list[Sense] = []
for s in senses:
if s not in out:
out.append(s)
return out
def _literal(tok: Token) -> Sense | None:
if tok.kind == 'domain':
return Sense('domain', tok.value)
if tok.kind == 'email':
return Sense('email_addr', tok.value)
if tok.kind == 'number' and tok.value is not None:
return Sense('number', tok.value)
if tok.kind == 'time' and tok.value is not None:
return Sense('time', tok.value)
if tok.kind == 'quoted':
return Sense('quoted', tok.value)
if tok.kind == 'path':
return Sense('path', tok.value)
if tok.kind == 'word' and EXT_ID.match(tok.text):
return Sense('ext_id', tok.text)
if tok.kind == 'word' and is_serial(tok.text):
return Sense('serial', tok.text.upper())
return None
def _fuzzy(tok: Token, lx: Lexicon, ov: _Overlay):
w = tok.text
if tok.kind != 'word' or len(w) < FUZZY_MIN or not w.isalpha() or w in lx.no_fuzzy or w in lx.words:
return None
# Group names in another word form: "engineers" -> Engineering, "marketers" -> Marketing.
forms = []
for target, senses in ov.fuzzy.items():
if not any(s.slot == 'group' for s in senses):
continue
cp = len(_common_prefix(w, target))
if cp >= 6 and cp >= 0.75 * min(len(w), len(target)):
forms.append((target, senses))
if len(forms) == 1:
target, senses = forms[0]
return _dedupe(senses), target, 0.85
limit = 1 if len(w) <= 7 else 2
best: list[tuple[int, str, list[Sense]]] = []
for pool in (ov.fuzzy, lx.fuzzy_targets):
for target, senses in pool.items():
if target[0] != w[0] or abs(len(target) - len(w)) > limit:
continue
d = _osa(w, target, limit)
if d <= limit:
best.append((d, target, senses))
if not best:
return None
best.sort(key=lambda x: x[0])
top = [b for b in best if b[0] == best[0][0]]
# Two different words equally close: guessing would be a coin toss, so do not.
if len({b[1] for b in top}) > 1:
return None
d, target, senses = top[0]
return _dedupe(senses), target, round(1 - d / max(len(w), 1), 2)
def _common_prefix(a: str, b: str) -> str:
n = 0
while n < min(len(a), len(b)) and a[n] == b[n]:
n += 1
return a[:n]
def _split_unknown_compounds(tokens: list[Token], lx: Lexicon) -> list[Token]:
"""'pen-drive' is not a vocabulary word, but 'pen drive' is: split hyphenated unknowns."""
out: list[Token] = []
for t in tokens:
if t.kind == 'word' and '-' in t.text and t.text not in lx.words and not is_serial(t.text):
pos = t.start
for part in t.text.split('-'):
if part:
out.append(Token(part, pos, pos + len(part), 'word'))
pos += len(part) + 1
else:
out.append(t)
return out